Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

initial cut at simpler producer API 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1410538 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-11-16 18:57:06 +00:00
parent c1ace1daf0
commit f711564486
11 changed files with 356 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

374
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
View File

@ -31,16 +31,20 @@ import org.apache.lucene.codecs.DocValuesArraySource;
import org.apache.lucene.codecs.NumericDocValuesConsumer;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.SimpleDocValuesFormat;
import org.apache.lucene.codecs.SortedDocValuesConsumer;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@ -76,7 +80,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
}
@Override
public PerDocProducer fieldsProducer(SegmentReadState state) throws IOException {
public SimpleDVProducer fieldsProducer(SegmentReadState state) throws IOException {
return new SimpleTextDocValuesReader(state.fieldInfos, state.dir, state.segmentInfo, state.context);
}
@ -328,7 +332,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
// nocommit make sure we test "all docs have 0 value",
// "all docs have empty BytesREf"
static class SimpleTextDocValuesReader extends PerDocProducer {
static class SimpleTextDocValuesReader extends SimpleDVProducer {
static class OneField {
FieldInfo fieldInfo;
@ -346,6 +350,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
final Map<String,OneField> fields = new HashMap<String,OneField>();
SimpleTextDocValuesReader(FieldInfos fieldInfos, Directory dir, SegmentInfo si, IOContext context) throws IOException {
super(si.getDocCount());
data = dir.openInput(IndexFileNames.segmentFileName(si.name, "", "dat"), context);
maxDoc = si.getDocCount();
while(true) {
@ -408,240 +413,150 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
}
}
class SimpleTextDocValues extends DocValues {
private final OneField field;
@Override
public NumericDocValues getDirectNumeric(FieldInfo fieldInfo) throws IOException {
final OneField field = fields.get(fieldInfo.name);
public SimpleTextDocValues(OneField field) {
this.field = field;
}
// SegmentCoreReaders already verifies this field is
// valid:
assert field != null;
// nocommit provide a simple default Source impl that
// loads DirectSource and pulls things into RAM; we
// need producer API to provide the min/max value,
// fixed/max length, etc.
final IndexInput in = data.clone();
final BytesRef scratch = new BytesRef();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
@Override
public Source loadSource() throws IOException {
DocValues.Type dvType = field.fieldInfo.getDocValuesType();
if (DocValues.isNumber(dvType)) {
Source source = loadDirectSource();
long[] values = new long[maxDoc];
for(int docID=0;docID<maxDoc;docID++) {
values[docID] = source.getInt(docID);
decoder.setParseBigDecimal(true);
return new NumericDocValues() {
@Override
public long get(int docID) {
try {
// nocommit bounds check docID? spooky
// because if we don't you can maybe get
// value from the wrong field ...
in.seek(field.dataStartFilePointer + (1+field.pattern.length())*docID);
SimpleTextUtil.readLine(in, scratch);
//System.out.println("parsing delta: " + scratch.utf8ToString());
BigDecimal bd;
try {
bd = (BigDecimal) decoder.parse(scratch.utf8ToString());
} catch (ParseException pe) {
CorruptIndexException e = new CorruptIndexException("failed to parse BigDecimal value");
e.initCause(pe);
throw e;
}
return BigInteger.valueOf(field.minValue).add(bd.toBigIntegerExact()).longValue();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
return DocValuesArraySource.forType(DocValues.Type.FIXED_INTS_64).newFromArray(values);
} else if (DocValues.isBytes(dvType)) {
Source source = loadDirectSource();
final byte[][] values = new byte[maxDoc][];
for(int docID=0;docID<maxDoc;docID++) {
// nocommit: who passes null!!!
BytesRef value = source.getBytes(docID, new BytesRef());
byte[] bytes = new byte[value.length];
System.arraycopy(value.bytes, value.offset, bytes, 0, value.length);
values[docID] = bytes;
}
return new Source(dvType) {
@Override
public BytesRef getBytes(int docID, BytesRef result) {
result.bytes = values[docID];
result.offset = 0;
result.length = result.bytes.length;
return result;
}
};
} else if (DocValues.isSortedBytes(dvType)) {
SortedSource source = (SortedSource) loadDirectSource();
final byte[][] values = new byte[field.numValues][];
BytesRef scratch = new BytesRef();
for(int ord=0;ord<field.numValues;ord++) {
source.getByOrd(ord, scratch);
values[ord] = new byte[scratch.length];
System.arraycopy(scratch.bytes, scratch.offset, values[ord], 0, scratch.length);
}
final int[] ords = new int[maxDoc];
for(int docID=0;docID<maxDoc;docID++) {
ords[docID] = source.ord(docID);
}
return new SortedSource(dvType, BytesRef.getUTF8SortedAsUnicodeComparator()) {
@Override
public int ord(int docID) {
return ords[docID];
}
@Override
public BytesRef getByOrd(int ord, BytesRef result) {
result.bytes = values[ord];
result.offset = 0;
result.length = result.bytes.length;
return result;
}
@Override
public int getValueCount() {
return field.numValues;
}
@Override
public PackedInts.Reader getDocToOrd() {
return null;
}
};
} else if (DocValues.isFloat(dvType)) {
// nocommit
return null;
} else {
throw new AssertionError();
}
}
@Override
public DocValues.Type getType() {
return field.fieldInfo.getDocValuesType();
}
@Override
public Source loadDirectSource() throws IOException {
DocValues.Type dvType = field.fieldInfo.getDocValuesType();
final IndexInput in = data.clone();
final BytesRef scratch = new BytesRef();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
if (DocValues.isNumber(dvType)) {
decoder.setParseBigDecimal(true);
return new Source(dvType) {
@Override
public long getInt(int docID) {
try {
// nocommit bounds check docID? spooky
// because if we don't you can maybe get
// value from the wrong field ...
in.seek(field.dataStartFilePointer + (1+field.pattern.length())*docID);
SimpleTextUtil.readLine(in, scratch);
//System.out.println("parsing delta: " + scratch.utf8ToString());
BigDecimal bd;
try {
bd = (BigDecimal) decoder.parse(scratch.utf8ToString());
} catch (ParseException pe) {
CorruptIndexException e = new CorruptIndexException("failed to parse BigDecimal value");
e.initCause(pe);
throw e;
}
return BigInteger.valueOf(field.minValue).add(bd.toBigIntegerExact()).longValue();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
};
} else if (DocValues.isBytes(dvType)) {
return new Source(dvType) {
@Override
public BytesRef getBytes(int docID, BytesRef result) {
try {
// nocommit bounds check docID? spooky
// because if we don't you can maybe get
// value from the wrong field ...
in.seek(field.dataStartFilePointer + (9+field.pattern.length() + field.maxLength)*docID);
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch, LENGTH);
int len;
try {
len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, "UTF-8")).intValue();
} catch (ParseException pe) {
// nocommit add message
CorruptIndexException e = new CorruptIndexException("failed to parse int length");
e.initCause(pe);
throw e;
}
result.bytes = new byte[len];
result.offset = 0;
result.length = len;
in.readBytes(result.bytes, 0, len);
return result;
} catch (IOException ioe) {
// nocommit should .get() just throw IOE...
throw new RuntimeException(ioe);
}
}
};
} else if (DocValues.isSortedBytes(dvType)) {
final DecimalFormat ordDecoder = new DecimalFormat(field.ordPattern, new DecimalFormatSymbols(Locale.ROOT));
return new SortedSource(dvType, BytesRef.getUTF8SortedAsUnicodeComparator()) {
@Override
public int ord(int docID) {
try {
in.seek(field.dataStartFilePointer + field.numValues * (9 + field.pattern.length() + field.maxLength) + docID * (1 + field.ordPattern.length()));
SimpleTextUtil.readLine(in, scratch);
try {
return ordDecoder.parse(scratch.utf8ToString()).intValue();
} catch (ParseException pe) {
CorruptIndexException e = new CorruptIndexException("failed to parse ord");
e.initCause(pe);
throw e;
}
} catch (IOException ioe) {
// nocommit should .get() just throw IOE...
throw new RuntimeException(ioe);
}
}
@Override
public BytesRef getByOrd(int ord, BytesRef result) {
try {
in.seek(field.dataStartFilePointer + ord * (9 + field.pattern.length() + field.maxLength));
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch, LENGTH);
int len;
try {
len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, "UTF-8")).intValue();
} catch (ParseException pe) {
CorruptIndexException e = new CorruptIndexException("failed to parse int length");
e.initCause(pe);
throw e;
}
result.bytes = new byte[len];
result.offset = 0;
result.length = len;
in.readBytes(result.bytes, 0, len);
return result;
} catch (IOException ioe) {
// nocommit should .get() just throw IOE...
throw new RuntimeException(ioe);
}
}
@Override
public int getValueCount() {
return field.numValues;
}
@Override
public PackedInts.Reader getDocToOrd() {
return null;
}
};
} else if (DocValues.isFloat(dvType)) {
// nocommit
return null;
} else {
throw new AssertionError();
}
}
};
}
@Override
public DocValues docValues(String fieldName) {
OneField field = fields.get(fieldName);
if (field == null) {
return null;
}
return new SimpleTextDocValues(field);
public BinaryDocValues getDirectBinary(FieldInfo fieldInfo) throws IOException {
final OneField field = fields.get(fieldInfo.name);
// SegmentCoreReaders already verifies this field is
// valid:
assert field != null;
final IndexInput in = data.clone();
final BytesRef scratch = new BytesRef();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
try {
// nocommit bounds check docID? spooky
// because if we don't you can maybe get
// value from the wrong field ...
in.seek(field.dataStartFilePointer + (9+field.pattern.length() + field.maxLength)*docID);
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch, LENGTH);
int len;
try {
len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, "UTF-8")).intValue();
} catch (ParseException pe) {
// nocommit add message
CorruptIndexException e = new CorruptIndexException("failed to parse int length");
e.initCause(pe);
throw e;
}
result.bytes = new byte[len];
result.offset = 0;
result.length = len;
in.readBytes(result.bytes, 0, len);
} catch (IOException ioe) {
// nocommit should .get() just throw IOE...
throw new RuntimeException(ioe);
}
}
};
}
@Override
public SortedDocValues getDirectSorted(FieldInfo fieldInfo) throws IOException {
final OneField field = fields.get(fieldInfo.name);
// SegmentCoreReaders already verifies this field is
// valid:
assert field != null;
final IndexInput in = data.clone();
final BytesRef scratch = new BytesRef();
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
final DecimalFormat ordDecoder = new DecimalFormat(field.ordPattern, new DecimalFormatSymbols(Locale.ROOT));
return new SortedDocValues() {
@Override
public int getOrd(int docID) {
try {
in.seek(field.dataStartFilePointer + field.numValues * (9 + field.pattern.length() + field.maxLength) + docID * (1 + field.ordPattern.length()));
SimpleTextUtil.readLine(in, scratch);
try {
return ordDecoder.parse(scratch.utf8ToString()).intValue();
} catch (ParseException pe) {
CorruptIndexException e = new CorruptIndexException("failed to parse ord");
e.initCause(pe);
throw e;
}
} catch (IOException ioe) {
// nocommit should .get() just throw IOE...
throw new RuntimeException(ioe);
}
}
@Override
public void lookupOrd(int ord, BytesRef result) {
try {
in.seek(field.dataStartFilePointer + ord * (9 + field.pattern.length() + field.maxLength));
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch, LENGTH);
int len;
try {
len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, "UTF-8")).intValue();
} catch (ParseException pe) {
CorruptIndexException e = new CorruptIndexException("failed to parse int length");
e.initCause(pe);
throw e;
}
result.bytes = new byte[len];
result.offset = 0;
result.length = len;
in.readBytes(result.bytes, 0, len);
} catch (IOException ioe) {
// nocommit should .get() just throw IOE...
throw new RuntimeException(ioe);
}
}
@Override
public int getValueCount() {
return field.numValues;
}
};
}
@Override
@ -649,14 +564,17 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
data.close();
}
/** Used only in ctor: */
private void readLine() throws IOException {
SimpleTextUtil.readLine(data, scratch);
}
/** Used only in ctor: */
private boolean startsWith(BytesRef prefix) {
return StringHelper.startsWith(scratch, prefix);
}
/** Used only in ctor: */
private String stripPrefix(BytesRef prefix) throws IOException {
return new String(scratch.bytes, scratch.offset + prefix.length, scratch.length - prefix.length, "UTF-8");
}

2
lucene/core/src/java/org/apache/lucene/codecs/SimpleDocValuesFormat.java
View File

@ -31,5 +31,5 @@ public abstract class SimpleDocValuesFormat {
public abstract SimpleDVConsumer fieldsConsumer(SegmentWriteState state) throws IOException;
// nocommit do this:
public abstract PerDocProducer fieldsProducer(SegmentReadState state) throws IOException;
public abstract SimpleDVProducer fieldsProducer(SegmentReadState state) throws IOException;
}

10
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SimpleDocValuesFormat.java
View File

@ -25,9 +25,10 @@ import java.util.TreeMap;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.PerDocProducerBase;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.SimpleDocValuesFormat;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@ -46,10 +47,13 @@ public class Lucene41SimpleDocValuesFormat extends SimpleDocValuesFormat {
}
@Override
public PerDocProducer fieldsProducer(SegmentReadState state)
public SimpleDVProducer fieldsProducer(SegmentReadState state)
throws IOException {
return new Lucene41PerDocProducer(state);
// nocommit fixme
//return new Lucene41PerDocProducer(state);
return null;
}
//nocommit this is equivalent to sep - we should pack in CFS
private static final class Lucene41PerDocProducer extends PerDocProducerBase {
private final TreeMap<String, DocValues> docValues;

11
lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
View File

@ -162,7 +162,16 @@ public abstract class AtomicReader extends IndexReader {
* values stored.
*/
public abstract DocValues docValues(String field) throws IOException;
// nocommit javadocs
public abstract NumericDocValues getNumericDocValues(String field, boolean direct) throws IOException;
// nocommit javadocs
public abstract BinaryDocValues getBinaryDocValues(String field, boolean direct) throws IOException;
// nocommit javadocs
public abstract SortedDocValues getSortedDocValues(String field, boolean direct) throws IOException;
/**
* Returns {@link DocValues} for this field's normalization values.
* This method may return null if the field has no norms.

3
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -605,7 +605,8 @@ public class CheckIndex {
// Test Term Vectors
segInfoStat.termVectorStatus = testTermVectors(fieldInfos, info, reader, nf);
// nocommit re-enable
segInfoStat.docValuesStatus = testDocValues(info, fieldInfos, reader);
// Rethrow the first exception we encountered

18
lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
View File

@ -410,6 +410,24 @@ public class FilterAtomicReader extends AtomicReader {
ensureOpen();
return in.docValues(field);
}
@Override
public NumericDocValues getNumericDocValues(String field, boolean direct) throws IOException {
ensureOpen();
return in.getNumericDocValues(field, direct);
}
@Override
public BinaryDocValues getBinaryDocValues(String field, boolean direct) throws IOException {
ensureOpen();
return in.getBinaryDocValues(field, direct);
}
@Override
public SortedDocValues getSortedDocValues(String field, boolean direct) throws IOException {
ensureOpen();
return in.getSortedDocValues(field, direct);
}
@Override
public DocValues normValues(String field) throws IOException {

21
lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
View File

@ -269,6 +269,27 @@ public final class ParallelAtomicReader extends AtomicReader {
AtomicReader reader = fieldToReader.get(field);
return reader == null ? null : reader.docValues(field);
}
@Override
public NumericDocValues getNumericDocValues(String field, boolean direct) throws IOException {
ensureOpen();
AtomicReader reader = fieldToReader.get(field);
return reader == null ? null : reader.getNumericDocValues(field, direct);
}
@Override
public BinaryDocValues getBinaryDocValues(String field, boolean direct) throws IOException {
ensureOpen();
AtomicReader reader = fieldToReader.get(field);
return reader == null ? null : reader.getBinaryDocValues(field, direct);
}
@Override
public SortedDocValues getSortedDocValues(String field, boolean direct) throws IOException {
ensureOpen();
AtomicReader reader = fieldToReader.get(field);
return reader == null ? null : reader.getSortedDocValues(field, direct);
}
@Override
public DocValues normValues(String field) throws IOException {

84
lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
View File

@ -19,7 +19,9 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@ -27,6 +29,7 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.index.SegmentReader.CoreClosedListener;
@ -51,6 +54,7 @@ final class SegmentCoreReaders {
final FieldInfos fieldInfos;
final FieldsProducer fields;
final SimpleDVProducer simpleDVProducer;
final PerDocProducer perDocProducer;
final PerDocProducer norms;
@ -62,6 +66,8 @@ final class SegmentCoreReaders {
final TermVectorsReader termVectorsReaderOrig;
final CompoundFileDirectory cfsReader;
private final Map<FieldInfo,Object> docValuesCache = new HashMap<FieldInfo,Object>();
final CloseableThreadLocal<StoredFieldsReader> fieldsReaderLocal = new CloseableThreadLocal<StoredFieldsReader>() {
@Override
protected StoredFieldsReader initialValue() {
@ -110,8 +116,9 @@ final class SegmentCoreReaders {
// TODO: since we don't write any norms file if there are no norms,
// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!
norms = codec.normsFormat().docsProducer(segmentReadState);
perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState);
// nocommit
perDocProducer = codec.simpleDocValuesFormat().fieldsProducer(segmentReadState);
simpleDVProducer = codec.simpleDocValuesFormat().fieldsProducer(segmentReadState);
fieldsReaderOrig = si.info.getCodec().storedFieldsFormat().fieldsReader(cfsDir, si.info, fieldInfos, context);
@ -138,12 +145,83 @@ final class SegmentCoreReaders {
void incRef() {
ref.incrementAndGet();
}
// nocommit shrink the sync'd part to a cache miss
synchronized NumericDocValues getNumericDocValues(String field, boolean direct) throws IOException {
FieldInfo fi = fieldInfos.fieldInfo(field);
if (fi == null) {
return null;
}
if (!DocValues.isNumber(fi.getDocValuesType())) {
throw new IllegalArgumentException("field \"" + field + "\" was not indexed as a numeric doc values field");
}
if (direct) {
return simpleDVProducer.getDirectNumeric(fi);
} else {
if (!docValuesCache.containsKey(fi)) {
NumericDocValues dv = simpleDVProducer.getNumeric(fi);
if (dv != null) {
docValuesCache.put(fi, dv);
}
}
return (NumericDocValues) docValuesCache.get(fi);
}
}
// nocommit shrink the sync'd part to a cache miss
synchronized BinaryDocValues getBinaryDocValues(String field, boolean direct) throws IOException {
FieldInfo fi = fieldInfos.fieldInfo(field);
if (fi == null) {
return null;
}
if (!DocValues.isBytes(fi.getDocValuesType())) {
throw new IllegalArgumentException("field \"" + field + "\" was not indexed as a binary doc values field");
}
if (direct) {
return simpleDVProducer.getDirectBinary(fi);
} else {
if (!docValuesCache.containsKey(fi)) {
BinaryDocValues dv = simpleDVProducer.getBinary(fi);
if (dv != null) {
docValuesCache.put(fi, dv);
}
}
return (BinaryDocValues) docValuesCache.get(fi);
}
}
// nocommit shrink the sync'd part to a cache miss
synchronized SortedDocValues getSortedDocValues(String field, boolean direct) throws IOException {
FieldInfo fi = fieldInfos.fieldInfo(field);
if (fi == null) {
return null;
}
if (!DocValues.isSortedBytes(fi.getDocValuesType())) {
throw new IllegalArgumentException("field \"" + field + "\" was not indexed as a sorted doc values field");
}
if (direct) {
return simpleDVProducer.getDirectSorted(fi);
} else {
if (!docValuesCache.containsKey(fi)) {
SortedDocValues dv = simpleDVProducer.getSorted(fi);
if (dv != null) {
docValuesCache.put(fi, dv);
}
}
return (SortedDocValues) docValuesCache.get(fi);
}
}
// nocommit binary, sorted too
void decRef() throws IOException {
//System.out.println("core.decRef seg=" + owner.getSegmentInfo() + " rc=" + ref);
if (ref.decrementAndGet() == 0) {
IOUtils.close(termVectorsLocal, fieldsReaderLocal, fields, perDocProducer,
termVectorsReaderOrig, fieldsReaderOrig, cfsReader, norms);
IOUtils.close(termVectorsLocal, fieldsReaderLocal, fields, simpleDVProducer,
perDocProducer, termVectorsReaderOrig, fieldsReaderOrig, cfsReader, norms);
notifyCoreClosedListeners();
}
}

15
lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -224,6 +224,21 @@ public final class SegmentReader extends AtomicReader {
public int getTermInfosIndexDivisor() {
return core.termsIndexDivisor;
}
@Override
public NumericDocValues getNumericDocValues(String field, boolean direct) throws IOException {
return core.getNumericDocValues(field, direct);
}
@Override
public BinaryDocValues getBinaryDocValues(String field, boolean direct) throws IOException {
return core.getBinaryDocValues(field, direct);
}
@Override
public SortedDocValues getSortedDocValues(String field, boolean direct) throws IOException {
return core.getSortedDocValues(field, direct);
}
@Override
public DocValues docValues(String field) throws IOException {

18
lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
View File

@ -87,6 +87,24 @@ public final class SlowCompositeReaderWrapper extends AtomicReader {
ensureOpen();
return MultiDocValues.getDocValues(in, field);
}
@Override
public NumericDocValues getNumericDocValues(String field, boolean direct) throws IOException {
// nocommit todo
return null;
}
@Override
public BinaryDocValues getBinaryDocValues(String field, boolean direct) throws IOException {
// nocommit todo
return null;
}
@Override
public SortedDocValues getSortedDocValues(String field, boolean direct) throws IOException {
// nocommit todo
return null;
}
@Override
public synchronized DocValues normValues(String field) throws IOException {

78
lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
View File

@ -27,12 +27,15 @@ import org.apache.lucene.document.PackedLongDocValuesField;
import org.apache.lucene.document.SortedBytesDocValuesField;
import org.apache.lucene.document.StraightBytesDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
@ -49,7 +52,7 @@ import org.junit.Ignore;
*/
public class TestDemoDocValue extends LuceneTestCase {
public void testDemo() throws IOException {
public void testDemoNumber() throws IOException {
Analyzer analyzer = new MockAnalyzer(random());
// Store the index in memory:
@ -79,21 +82,15 @@ public class TestDemoDocValue extends LuceneTestCase {
StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
assertEquals(5, dv.getSource().getInt(hits.scoreDocs[i].doc));
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv", random().nextBoolean());
assertEquals(5, dv.get(hits.scoreDocs[i].doc));
}
// Test simple phrase query
PhraseQuery phraseQuery = new PhraseQuery();
phraseQuery.add(new Term("fieldname", "to"));
phraseQuery.add(new Term("fieldname", "be"));
assertEquals(1, isearcher.search(phraseQuery, null, 1).totalHits);
ireader.close();
directory.close();
}
public void testTwoDocuments() throws IOException {
public void testTwoDocumentsNumeric() throws IOException {
Analyzer analyzer = new MockAnalyzer(random());
// Store the index in memory:
@ -116,9 +113,9 @@ public class TestDemoDocValue extends LuceneTestCase {
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
assertEquals(1, dv.getSource().getInt(0));
assertEquals(2, dv.getSource().getInt(1));
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv", random().nextBoolean());
assertEquals(1, dv.get(0));
assertEquals(2, dv.get(1));
ireader.close();
directory.close();
@ -150,7 +147,7 @@ public class TestDemoDocValue extends LuceneTestCase {
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv", random().nextBoolean());
for(int i=0;i<2;i++) {
StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);
long expected;
@ -159,7 +156,7 @@ public class TestDemoDocValue extends LuceneTestCase {
} else {
expected = 3;
}
assertEquals(expected, dv.getSource().getInt(i));
assertEquals(expected, dv.get(i));
}
ireader.close();
@ -189,9 +186,9 @@ public class TestDemoDocValue extends LuceneTestCase {
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
assertEquals(Long.MIN_VALUE, dv.getSource().getInt(0));
assertEquals(Long.MAX_VALUE, dv.getSource().getInt(1));
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv", random().nextBoolean());
assertEquals(Long.MIN_VALUE, dv.get(0));
assertEquals(Long.MAX_VALUE, dv.get(1));
ireader.close();
directory.close();
@ -222,21 +219,17 @@ public class TestDemoDocValue extends LuceneTestCase {
Query query = new TermQuery(new Term("fieldname", "text"));
TopDocs hits = isearcher.search(query, null, 1);
assertEquals(1, hits.totalHits);
BytesRef scratch = new BytesRef();
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
assertEquals(new BytesRef("hello world"), dv.getSource().getBytes(hits.scoreDocs[i].doc, new BytesRef()));
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv", random().nextBoolean());
dv.get(hits.scoreDocs[i].doc, scratch);
assertEquals(new BytesRef("hello world"), scratch);
}
// Test simple phrase query
PhraseQuery phraseQuery = new PhraseQuery();
phraseQuery.add(new Term("fieldname", "to"));
phraseQuery.add(new Term("fieldname", "be"));
assertEquals(1, isearcher.search(phraseQuery, null, 1).totalHits);
ireader.close();
directory.close();
@ -269,7 +262,8 @@ public class TestDemoDocValue extends LuceneTestCase {
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv", random().nextBoolean());
BytesRef scratch = new BytesRef();
for(int i=0;i<2;i++) {
StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);
String expected;
@ -278,7 +272,8 @@ public class TestDemoDocValue extends LuceneTestCase {
} else {
expected = "hello world 2";
}
assertEquals(expected, dv.getSource().getBytes(i, new BytesRef()).utf8ToString());
dv.get(i, scratch);
assertEquals(expected, scratch.utf8ToString());
}
ireader.close();
@ -310,21 +305,17 @@ public class TestDemoDocValue extends LuceneTestCase {
Query query = new TermQuery(new Term("fieldname", "text"));
TopDocs hits = isearcher.search(query, null, 1);
assertEquals(1, hits.totalHits);
BytesRef scratch = new BytesRef();
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
assertEquals(new BytesRef("hello world"), dv.getSource().getBytes(hits.scoreDocs[i].doc, new BytesRef()));
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv", random().nextBoolean());
dv.lookupOrd(dv.getOrd(hits.scoreDocs[i].doc), scratch);
assertEquals(new BytesRef("hello world"), scratch);
}
// Test simple phrase query
PhraseQuery phraseQuery = new PhraseQuery();
phraseQuery.add(new Term("fieldname", "to"));
phraseQuery.add(new Term("fieldname", "be"));
assertEquals(1, isearcher.search(phraseQuery, null, 1).totalHits);
ireader.close();
directory.close();
}
@ -352,9 +343,12 @@ public class TestDemoDocValue extends LuceneTestCase {
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
assertEquals("hello world 1", dv.getSource().getBytes(0, new BytesRef()).utf8ToString());
assertEquals("hello world 2", dv.getSource().getBytes(1, new BytesRef()).utf8ToString());
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv", random().nextBoolean());
BytesRef scratch = new BytesRef();
dv.lookupOrd(dv.getOrd(0), scratch);
assertEquals("hello world 1", scratch.utf8ToString());
dv.lookupOrd(dv.getOrd(1), scratch);
assertEquals("hello world 2", scratch.utf8ToString());
ireader.close();
directory.close();
@ -386,7 +380,8 @@ public class TestDemoDocValue extends LuceneTestCase {
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
DocValues dv = ireader.leaves().get(0).reader().docValues("dv");
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv", random().nextBoolean());
BytesRef scratch = new BytesRef();
for(int i=0;i<2;i++) {
StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);
String expected;
@ -395,7 +390,8 @@ public class TestDemoDocValue extends LuceneTestCase {
} else {
expected = "hello world 2";
}
assertEquals(expected, dv.getSource().getBytes(i, new BytesRef()).utf8ToString());
dv.lookupOrd(dv.getOrd(i), scratch);
assertEquals(expected, scratch.utf8ToString());
}
ireader.close();