Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-07 18:49:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

make clone() explicit, move responsibility out of codec into SCR

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1416914 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-12-04 12:37:08 +00:00
parent 1ef8f5e966
commit 47632b0b82
9 changed files with 108 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
View File

@ -132,6 +132,7 @@ public class MemoryDocValuesFormat extends SimpleDocValuesFormat {
};
}
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
SortedDocValues valuesIn = producer.getSorted(field);
final int maxDoc = valuesIn.size();
@ -188,6 +189,12 @@ public class MemoryDocValuesFormat extends SimpleDocValuesFormat {
};
}
@Override
public SimpleDVProducer clone() {
// We are already thread-safe:
return this;
}
@Override
public void close() throws IOException {
producer.close();

18
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
View File

@ -487,6 +487,17 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
assert !fields.isEmpty();
}
private SimpleTextDocValuesReader(SimpleTextDocValuesReader other) {
this.data = other.data.clone();
this.fields.putAll(other.fields);
this.maxDoc = other.maxDoc;
}
@Override
public SimpleDVProducer clone() {
return new SimpleTextDocValuesReader(this);
}
@Override
public NumericDocValues getNumeric(FieldInfo fieldInfo) throws IOException {
final OneField field = fields.get(fieldInfo.name);
@ -505,7 +516,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
// valid:
assert field != null: "field=" + fieldInfo.name + " fields=" + fields;
final IndexInput in = data.clone();
final IndexInput in = data;
final BytesRef scratch = new BytesRef();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
@ -515,6 +526,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
@Override
public long get(int docID) {
try {
//System.out.println(Thread.currentThread().getName() + ": get docID=" + docID + " in=" + in);
if (docID < 0 || docID >= maxDoc) {
throw new IndexOutOfBoundsException("docID must be 0 .. " + (maxDoc-1) + "; got " + docID);
}
@ -560,7 +572,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
// valid:
assert field != null;
final IndexInput in = data.clone();
final IndexInput in = data;
final BytesRef scratch = new BytesRef();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
@ -616,7 +628,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
// valid:
assert field != null;
final IndexInput in = data.clone();
final IndexInput in = data;
final BytesRef scratch = new BytesRef();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
final DecimalFormat ordDecoder = new DecimalFormat(field.ordPattern, new DecimalFormatSymbols(Locale.ROOT));

2
lucene/core/src/java/org/apache/lucene/codecs/SimpleDVProducer.java
View File

@ -37,4 +37,6 @@ public abstract class SimpleDVProducer implements Closeable {
public abstract BinaryDocValues getBinary(FieldInfo field) throws IOException;
public abstract SortedDocValues getSorted(FieldInfo field) throws IOException;
public abstract SimpleDVProducer clone();
}

6
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
View File

@ -81,6 +81,12 @@ public class Lucene41DocValuesProducer extends SimpleDVProducer {
IOUtils.close(cfs);
}
}
@Override
public SimpleDVProducer clone() {
// nocommit todo
return null;
}
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {

27
lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.codecs.perfield;
import java.io.Closeable;
import java.io.IOException;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.ServiceLoader; // javadocs
import java.util.TreeMap;
@ -183,6 +184,9 @@ public abstract class PerFieldDocValuesFormat extends SimpleDocValuesFormat {
}
}
// nocommit what if SimpleNormsFormat wants to use this
// ...? we have a "boolean isNorms" issue...? I guess we
// just need to make a PerFieldNormsFormat?
private class FieldsReader extends SimpleDVProducer {
private final Map<String,SimpleDVProducer> fields = new TreeMap<String,SimpleDVProducer>();
@ -219,6 +223,24 @@ public abstract class PerFieldDocValuesFormat extends SimpleDocValuesFormat {
}
}
private FieldsReader(FieldsReader other) {
Map<SimpleDVProducer,SimpleDVProducer> oldToNew = new IdentityHashMap<SimpleDVProducer,SimpleDVProducer>();
// First clone all formats
for(Map.Entry<String,SimpleDVProducer> ent : other.formats.entrySet()) {
SimpleDVProducer values = ent.getValue().clone();
formats.put(ent.getKey(), values);
oldToNew.put(ent.getValue(), values);
}
// Then rebuild fields:
for(Map.Entry<String,SimpleDVProducer> ent : other.fields.entrySet()) {
SimpleDVProducer producer = oldToNew.get(ent.getValue());
assert producer != null;
fields.put(ent.getKey(), producer);
}
}
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
SimpleDVProducer producer = fields.get(field.name);
@ -241,6 +263,11 @@ public abstract class PerFieldDocValuesFormat extends SimpleDocValuesFormat {
public void close() throws IOException {
IOUtils.close(formats.values());
}
@Override
public SimpleDVProducer clone() {
return new FieldsReader(this);
}
}
@Override

4
lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
View File

@ -163,16 +163,19 @@ public abstract class AtomicReader extends IndexReader {
*/
public abstract DocValues docValues(String field) throws IOException;
// nocommit document that these are thread-private:
/** Returns {@link NumericDocValues} for this field, or
* null if no {@link NumericDocValues} were indexed for
* this field. */
public abstract NumericDocValues getNumericDocValues(String field) throws IOException;
// nocommit document that these are thread-private:
/** Returns {@link BinaryDocValues} for this field, or
* null if no {@link BinaryDocValues} were indexed for
* this field. */
public abstract BinaryDocValues getBinaryDocValues(String field) throws IOException;
// nocommit document that these are thread-private:
/** Returns {@link SortedDocValues} for this field, or
* null if no {@link SortedDocValues} were indexed for
* this field. */
@ -184,6 +187,7 @@ public abstract class AtomicReader extends IndexReader {
*/
public abstract DocValues normValues(String field) throws IOException;
// nocommit document that these are thread-private:
/** Returns {@link NumericDocValues} representing norms
* for this field, or null if no {@link NumericDocValues}
* were indexed. */

38
lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
View File

@ -65,6 +65,9 @@ final class SegmentCoreReaders {
final TermVectorsReader termVectorsReaderOrig;
final CompoundFileDirectory cfsReader;
// nocommit we should make a single thread local w/ a
// class holding these N things...?
final CloseableThreadLocal<StoredFieldsReader> fieldsReaderLocal = new CloseableThreadLocal<StoredFieldsReader>() {
@Override
protected StoredFieldsReader initialValue() {
@ -75,10 +78,31 @@ final class SegmentCoreReaders {
final CloseableThreadLocal<TermVectorsReader> termVectorsLocal = new CloseableThreadLocal<TermVectorsReader>() {
@Override
protected TermVectorsReader initialValue() {
return (termVectorsReaderOrig == null) ?
null : termVectorsReaderOrig.clone();
return (termVectorsReaderOrig == null) ? null : termVectorsReaderOrig.clone();
}
};
// nocommit not great to hold onto lots-o-ram in a thread
// local...? do we need a "needsClone"/"isThreadSafe"!?
final CloseableThreadLocal<SimpleDVProducer> simpleDocValuesLocal = new CloseableThreadLocal<SimpleDVProducer>() {
@Override
protected SimpleDVProducer initialValue() {
// nocommit remove null check
return (simpleDVProducer == null) ? null : simpleDVProducer.clone();
}
};
// nocommit not great to hold onto lots-o-ram in a thread
// local...? do we need a "needsClone"/"isThreadSafe"!?
final CloseableThreadLocal<SimpleDVProducer> simpleNormsLocal = new CloseableThreadLocal<SimpleDVProducer>() {
@Override
protected SimpleDVProducer initialValue() {
// nocommit remove null check
return (simpleNormsProducer == null) ? null : simpleNormsProducer.clone();
}
};
// nocommit norms too
private final Set<CoreClosedListener> coreClosedListeners =
Collections.synchronizedSet(new LinkedHashSet<CoreClosedListener>());
@ -181,7 +205,7 @@ final class SegmentCoreReaders {
return null;
}
return simpleDVProducer.getNumeric(fi);
return simpleDocValuesLocal.get().getNumeric(fi);
}
BinaryDocValues getBinaryDocValues(String field) throws IOException {
@ -204,7 +228,7 @@ final class SegmentCoreReaders {
return null;
}
return simpleDVProducer.getBinary(fi);
return simpleDocValuesLocal.get().getBinary(fi);
}
SortedDocValues getSortedDocValues(String field) throws IOException {
@ -227,7 +251,7 @@ final class SegmentCoreReaders {
return null;
}
return simpleDVProducer.getSorted(fi);
return simpleDocValuesLocal.get().getSorted(fi);
}
NumericDocValues getSimpleNormValues(String field) throws IOException {
@ -243,12 +267,12 @@ final class SegmentCoreReaders {
if (simpleNormsProducer == null) {
return null;
}
return simpleNormsProducer.getNumeric(fi);
return simpleNormsLocal.get().getNumeric(fi);
}
void decRef() throws IOException {
if (ref.decrementAndGet() == 0) {
IOUtils.close(termVectorsLocal, fieldsReaderLocal, fields, simpleDVProducer,
IOUtils.close(termVectorsLocal, fieldsReaderLocal, simpleDocValuesLocal, simpleNormsLocal, fields, simpleDVProducer,
perDocProducer, termVectorsReaderOrig, fieldsReaderOrig, cfsReader, norms,
simpleNormsProducer);
notifyCoreClosedListeners();

11
lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
View File

@ -986,8 +986,6 @@ public class TestDocValuesIndexing extends LuceneTestCase {
writer.close();
final AtomicReader sr = getOnlySegmentReader(r);
final SortedDocValues dv = sorted(sr, "stringdv");
assertNotNull(dv);
final long END_TIME = System.currentTimeMillis() + (TEST_NIGHTLY ? 30 : 1);
@ -1000,11 +998,8 @@ public class TestDocValuesIndexing extends LuceneTestCase {
@Override
public void run() {
Random random = random();
final SortedDocValues stringDV = dv;
final SortedDocValues stringDVDirect;
try {
assertNotNull(stringDV);
stringDVDirect = sr.getSortedDocValues("stringdv");
assertNotNull(stringDVDirect);
} catch (IOException ioe) {
@ -1012,11 +1007,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
}
while(System.currentTimeMillis() < END_TIME) {
final SortedDocValues source;
if (random.nextBoolean()) {
source = stringDV;
} else {
source = stringDVDirect;
}
source = stringDVDirect;
final BytesRef scratch = new BytesRef();
for(int iter=0;iter<100;iter++) {

23
lucene/core/src/test/org/apache/lucene/search/TestSameScoresWithThreads.java
View File

@ -49,24 +49,31 @@ public class TestSameScoresWithThreads extends LuceneTestCase {
final Directory dir = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
LineFileDocs docs = new LineFileDocs(random());
int bytesToIndex = atLeast(100000);
int bytesIndexed = 0;
while(bytesIndexed < bytesToIndex) {
int charsToIndex = atLeast(100000);
int charsIndexed = 0;
//System.out.println("bytesToIndex=" + charsToIndex);
while(charsIndexed < charsToIndex) {
Document doc = docs.nextDoc();
bytesIndexed += RamUsageEstimator.sizeOf(doc);
charsIndexed += doc.get("body").length();
w.addDocument(doc);
//System.out.println(" bytes=" + charsIndexed + " add: " + doc);
}
IndexReader r = w.getReader();
//System.out.println("numDocs=" + r.numDocs());
w.close();
final IndexSearcher s = new IndexSearcher(r);
Terms terms = MultiFields.getFields(r).terms("body");
long termCount = terms.size();
int termCount = 0;
TermsEnum termsEnum = terms.iterator(null);
while(termsEnum.next() != null) {
termCount++;
}
assertTrue(termCount > 0);
// Target ~10 terms to search:
double chance = 10.0 / termCount;
TermsEnum termsEnum = terms.iterator(null);
termsEnum = terms.iterator(termsEnum);
final Map<BytesRef,TopDocs> answers = new HashMap<BytesRef,TopDocs>();
while(termsEnum.next() != null) {
if (random().nextDouble() <= chance) {
@ -90,10 +97,10 @@ public class TestSameScoresWithThreads extends LuceneTestCase {
List<Map.Entry<BytesRef,TopDocs>> shuffled = new ArrayList<Map.Entry<BytesRef,TopDocs>>(answers.entrySet());
Collections.shuffle(shuffled);
for(Map.Entry<BytesRef,TopDocs> ent : shuffled) {
TopDocs actual = s.search(new TermQuery(new Term("body", ent.getKey())), 10);
TopDocs actual = s.search(new TermQuery(new Term("body", ent.getKey())), 100);
TopDocs expected = ent.getValue();
assertEquals(expected.totalHits, actual.totalHits);
assertEquals(expected.scoreDocs.length, actual.scoreDocs.length);
assertEquals("query=" + ent.getKey().utf8ToString(), expected.scoreDocs.length, actual.scoreDocs.length);
for(int hit=0;hit<expected.scoreDocs.length;hit++) {
assertEquals(expected.scoreDocs[hit].doc, actual.scoreDocs[hit].doc);
// Floats really should be identical: