Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3679: replace IR.getFieldNames with IR.getFieldInfos 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1229401 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-01-09 22:29:40 +00:00
parent 5dba63c719
commit cdb2ee8a7b
41 changed files with 354 additions and 508 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/CHANGES.txt
View File

@ -720,6 +720,11 @@ Changes in backwards compatibility policy
break in the binary format so if you had built and saved any FSTs break in the binary format so if you had built and saved any FSTs
then you need to rebuild them. (Robert Muir, Mike McCandless) then you need to rebuild them. (Robert Muir, Mike McCandless)
* LUCENE-3679: The expert IndexReader.getFieldNames(FieldOption) API
has been removed and replaced with the experimental getFieldInfos
API. All IndexReader subclasses must implement getFieldInfos.
(Mike McCandless)
Security fixes Security fixes
* LUCENE-3588: Try harder to prevent SIGSEGV on cloned MMapIndexInputs: * LUCENE-3588: Try harder to prevent SIGSEGV on cloned MMapIndexInputs:

2
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
View File

@ -128,8 +128,6 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
public void stringField(FieldInfo fieldInfo, String value) throws IOException { public void stringField(FieldInfo fieldInfo, String value) throws IOException {
FieldType ft = new FieldType(TextField.TYPE_STORED); FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.storeTermVector); ft.setStoreTermVectors(fieldInfo.storeTermVector);
ft.setStoreTermVectorOffsets(fieldInfo.storeOffsetWithTermVector);
ft.setStoreTermVectorPositions(fieldInfo.storePositionWithTermVector);
fields.add(new Field(fieldInfo.name, value, ft)); fields.add(new Field(fieldInfo.name, value, ft));
} }

30
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -21,7 +21,6 @@ import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
@ -34,13 +33,14 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.Fields; import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.OrdTermState; import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.StoredFieldVisitor;
@ -198,6 +198,8 @@ public class MemoryIndex {
private static final boolean DEBUG = false; private static final boolean DEBUG = false;
private final FieldInfos fieldInfos;
/** /**
* Sorts term entries into ascending order; also works for * Sorts term entries into ascending order; also works for
* Arrays.binarySearch() and Arrays.sort() * Arrays.binarySearch() and Arrays.sort()
@ -232,6 +234,7 @@ public class MemoryIndex {
*/ */
private MemoryIndex(boolean storeOffsets) { private MemoryIndex(boolean storeOffsets) {
this.stride = storeOffsets ? 3 : 1; this.stride = storeOffsets ? 3 : 1;
fieldInfos = new FieldInfos();
} }
/** /**
@ -348,6 +351,8 @@ public class MemoryIndex {
int numOverlapTokens = 0; int numOverlapTokens = 0;
int pos = -1; int pos = -1;
fieldInfos.addOrUpdate(fieldName, true);
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class); PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class); OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
@ -765,6 +770,11 @@ public class MemoryIndex {
return null; return null;
} }
@Override
public FieldInfos getFieldInfos() {
return fieldInfos;
}
@Override @Override
public int docFreq(String field, BytesRef term) { public int docFreq(String field, BytesRef term) {
Info info = getInfo(field); Info info = getInfo(field);
@ -1113,22 +1123,6 @@ public class MemoryIndex {
if (DEBUG) System.err.println("MemoryIndexReader.doClose"); if (DEBUG) System.err.println("MemoryIndexReader.doClose");
} }
// lucene >= 1.9 (remove this method for lucene-1.4.3)
@Override
public Collection<String> getFieldNames(FieldOption fieldOption) {
if (DEBUG) System.err.println("MemoryIndexReader.getFieldNamesOption");
if (fieldOption == FieldOption.UNINDEXED)
return Collections.<String>emptySet();
if (fieldOption == FieldOption.INDEXED_NO_TERMVECTOR)
return Collections.<String>emptySet();
if (fieldOption == FieldOption.TERMVECTOR_WITH_OFFSET && stride == 1)
return Collections.<String>emptySet();
if (fieldOption == FieldOption.TERMVECTOR_WITH_POSITION_OFFSET && stride == 1)
return Collections.<String>emptySet();
return Collections.unmodifiableSet(fields.keySet());
}
@Override @Override
public DocValues docValues(String field) throws IOException { public DocValues docValues(String field) throws IOException {
return null; return null;

3
lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
View File

@ -69,8 +69,6 @@ public class Lucene40FieldInfosReader extends FieldInfosReader {
byte bits = input.readByte(); byte bits = input.readByte();
boolean isIndexed = (bits & Lucene40FieldInfosWriter.IS_INDEXED) != 0; boolean isIndexed = (bits & Lucene40FieldInfosWriter.IS_INDEXED) != 0;
boolean storeTermVector = (bits & Lucene40FieldInfosWriter.STORE_TERMVECTOR) != 0; boolean storeTermVector = (bits & Lucene40FieldInfosWriter.STORE_TERMVECTOR) != 0;
boolean storePositionsWithTermVector = (bits & Lucene40FieldInfosWriter.STORE_POSITIONS_WITH_TERMVECTOR) != 0;
boolean storeOffsetWithTermVector = (bits & Lucene40FieldInfosWriter.STORE_OFFSET_WITH_TERMVECTOR) != 0;
boolean omitNorms = (bits & Lucene40FieldInfosWriter.OMIT_NORMS) != 0; boolean omitNorms = (bits & Lucene40FieldInfosWriter.OMIT_NORMS) != 0;
boolean storePayloads = (bits & Lucene40FieldInfosWriter.STORE_PAYLOADS) != 0; boolean storePayloads = (bits & Lucene40FieldInfosWriter.STORE_PAYLOADS) != 0;
final IndexOptions indexOptions; final IndexOptions indexOptions;
@ -147,7 +145,6 @@ public class Lucene40FieldInfosReader extends FieldInfosReader {
} }
} }
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
storePositionsWithTermVector, storeOffsetWithTermVector,
omitNorms, storePayloads, indexOptions, docValuesType); omitNorms, storePayloads, indexOptions, docValuesType);
} }

9
lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosWriter.java
View File

@ -47,8 +47,6 @@ public class Lucene40FieldInfosWriter extends FieldInfosWriter {
static final byte IS_INDEXED = 0x1; static final byte IS_INDEXED = 0x1;
static final byte STORE_TERMVECTOR = 0x2; static final byte STORE_TERMVECTOR = 0x2;
static final byte STORE_POSITIONS_WITH_TERMVECTOR = 0x4;
static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x8;
static final byte OMIT_NORMS = 0x10; static final byte OMIT_NORMS = 0x10;
static final byte STORE_PAYLOADS = 0x20; static final byte STORE_PAYLOADS = 0x20;
static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40; static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40;
@ -66,14 +64,13 @@ public class Lucene40FieldInfosWriter extends FieldInfosWriter {
byte bits = 0x0; byte bits = 0x0;
if (fi.isIndexed) bits |= IS_INDEXED; if (fi.isIndexed) bits |= IS_INDEXED;
if (fi.storeTermVector) bits |= STORE_TERMVECTOR; if (fi.storeTermVector) bits |= STORE_TERMVECTOR;
if (fi.storePositionWithTermVector) bits |= STORE_POSITIONS_WITH_TERMVECTOR;
if (fi.storeOffsetWithTermVector) bits |= STORE_OFFSET_WITH_TERMVECTOR;
if (fi.omitNorms) bits |= OMIT_NORMS; if (fi.omitNorms) bits |= OMIT_NORMS;
if (fi.storePayloads) bits |= STORE_PAYLOADS; if (fi.storePayloads) bits |= STORE_PAYLOADS;
if (fi.indexOptions == IndexOptions.DOCS_ONLY) if (fi.indexOptions == IndexOptions.DOCS_ONLY) {
bits |= OMIT_TERM_FREQ_AND_POSITIONS; bits |= OMIT_TERM_FREQ_AND_POSITIONS;
else if (fi.indexOptions == IndexOptions.DOCS_AND_FREQS) } else if (fi.indexOptions == IndexOptions.DOCS_AND_FREQS) {
bits |= OMIT_POSITIONS; bits |= OMIT_POSITIONS;
}
output.writeString(fi.name); output.writeString(fi.name);
output.writeInt(fi.number); output.writeInt(fi.number);
output.writeByte(bits); output.writeByte(bits);

9
lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
View File

@ -79,14 +79,6 @@ public class SimpleTextFieldInfosReader extends FieldInfosReader {
assert StringHelper.startsWith(scratch, STORETV); assert StringHelper.startsWith(scratch, STORETV);
boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch)); boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch));
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch, STORETVPOS);
boolean storePositionsWithTermVector = Boolean.parseBoolean(readString(STORETVPOS.length, scratch));
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch, STORETVOFF);
boolean storeOffsetWithTermVector = Boolean.parseBoolean(readString(STORETVOFF.length, scratch));
SimpleTextUtil.readLine(input, scratch); SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch, PAYLOADS); assert StringHelper.startsWith(scratch, PAYLOADS);
boolean storePayloads = Boolean.parseBoolean(readString(PAYLOADS.length, scratch)); boolean storePayloads = Boolean.parseBoolean(readString(PAYLOADS.length, scratch));
@ -115,7 +107,6 @@ public class SimpleTextFieldInfosReader extends FieldInfosReader {
hasFreq |= isIndexed && indexOptions != IndexOptions.DOCS_ONLY; hasFreq |= isIndexed && indexOptions != IndexOptions.DOCS_ONLY;
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
storePositionsWithTermVector, storeOffsetWithTermVector,
omitNorms, storePayloads, indexOptions, docValuesType); omitNorms, storePayloads, indexOptions, docValuesType);
} }

8
lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
View File

@ -80,14 +80,6 @@ public class SimpleTextFieldInfosWriter extends FieldInfosWriter {
SimpleTextUtil.write(out, Boolean.toString(fi.storeTermVector), scratch); SimpleTextUtil.write(out, Boolean.toString(fi.storeTermVector), scratch);
SimpleTextUtil.writeNewline(out); SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, STORETVPOS);
SimpleTextUtil.write(out, Boolean.toString(fi.storePositionWithTermVector), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, STORETVOFF);
SimpleTextUtil.write(out, Boolean.toString(fi.storeOffsetWithTermVector), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, PAYLOADS); SimpleTextUtil.write(out, PAYLOADS);
SimpleTextUtil.write(out, Boolean.toString(fi.storePayloads), scratch); SimpleTextUtil.write(out, Boolean.toString(fi.storePayloads), scratch);
SimpleTextUtil.writeNewline(out); SimpleTextUtil.writeNewline(out);

2
lucene/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java
View File

@ -64,8 +64,6 @@ public class DocumentStoredFieldVisitor extends StoredFieldVisitor {
public void stringField(FieldInfo fieldInfo, String value) throws IOException { public void stringField(FieldInfo fieldInfo, String value) throws IOException {
final FieldType ft = new FieldType(TextField.TYPE_STORED); final FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.storeTermVector); ft.setStoreTermVectors(fieldInfo.storeTermVector);
ft.setStoreTermVectorPositions(fieldInfo.storePositionWithTermVector);
ft.setStoreTermVectorOffsets(fieldInfo.storeOffsetWithTermVector);
ft.setStoreTermVectors(fieldInfo.storeTermVector); ft.setStoreTermVectors(fieldInfo.storeTermVector);
ft.setIndexed(fieldInfo.isIndexed); ft.setIndexed(fieldInfo.isIndexed);
ft.setOmitNorms(fieldInfo.omitNorms); ft.setOmitNorms(fieldInfo.omitNorms);

19
lucene/src/java/org/apache/lucene/index/BaseMultiReader.java
View File

@ -18,9 +18,6 @@ package org.apache.lucene.index;
*/ */
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
@ -55,6 +52,11 @@ abstract class BaseMultiReader<R extends IndexReader> extends IndexReader {
topLevelContext = ReaderUtil.buildReaderContext(this); topLevelContext = ReaderUtil.buildReaderContext(this);
} }
@Override
public FieldInfos getFieldInfos() {
throw new UnsupportedOperationException("call getFieldInfos() on each sub reader, or use ReaderUtil.getMergedFieldInfos, instead");
}
@Override @Override
public Fields fields() throws IOException { public Fields fields() throws IOException {
throw new UnsupportedOperationException("please use MultiFields.getFields, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Fields"); throw new UnsupportedOperationException("please use MultiFields.getFields, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Fields");
@ -127,17 +129,6 @@ abstract class BaseMultiReader<R extends IndexReader> extends IndexReader {
return total; return total;
} }
@Override
public Collection<String> getFieldNames (IndexReader.FieldOption fieldNames) {
ensureOpen();
// maintain a unique set of field names
final Set<String> fieldSet = new HashSet<String>();
for (IndexReader reader : subReaders) {
fieldSet.addAll(reader.getFieldNames(fieldNames));
}
return fieldSet;
}
@Override @Override
public IndexReader[] getSequentialSubReaders() { public IndexReader[] getSequentialSubReaders() {
return subReaders; return subReaders;

31
lucene/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -23,7 +23,6 @@ import java.io.PrintStream;
import java.lang.reflect.Array; import java.lang.reflect.Array;
import java.text.NumberFormat; import java.text.NumberFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -552,16 +551,16 @@ public class CheckIndex {
if (reader.maxDoc() != info.docCount) if (reader.maxDoc() != info.docCount)
throw new RuntimeException("SegmentReader.maxDoc() " + reader.maxDoc() + " != SegmentInfos.docCount " + info.docCount); throw new RuntimeException("SegmentReader.maxDoc() " + reader.maxDoc() + " != SegmentInfos.docCount " + info.docCount);
// Test getFieldNames() // Test getFieldInfos()
if (infoStream != null) { if (infoStream != null) {
infoStream.print(" test: fields.............."); infoStream.print(" test: fields..............");
} }
Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL); FieldInfos fieldInfos = reader.getFieldInfos();
msg("OK [" + fieldNames.size() + " fields]"); msg("OK [" + fieldInfos.size() + " fields]");
segInfoStat.numFields = fieldNames.size(); segInfoStat.numFields = fieldInfos.size();
// Test Field Norms // Test Field Norms
segInfoStat.fieldNormStatus = testFieldNorms(fieldNames, reader); segInfoStat.fieldNormStatus = testFieldNorms(fieldInfos, reader);
// Test the Term Index // Test the Term Index
segInfoStat.termIndexStatus = testTermIndex(reader); segInfoStat.termIndexStatus = testTermIndex(reader);
@ -631,7 +630,7 @@ public class CheckIndex {
/** /**
* Test field norms. * Test field norms.
*/ */
private Status.FieldNormStatus testFieldNorms(Collection<String> fieldNames, SegmentReader reader) { private Status.FieldNormStatus testFieldNorms(FieldInfos fieldInfos, SegmentReader reader) {
final Status.FieldNormStatus status = new Status.FieldNormStatus(); final Status.FieldNormStatus status = new Status.FieldNormStatus();
try { try {
@ -639,29 +638,27 @@ public class CheckIndex {
if (infoStream != null) { if (infoStream != null) {
infoStream.print(" test: field norms........."); infoStream.print(" test: field norms.........");
} }
FieldInfos infos = reader.fieldInfos();
DocValues dv; DocValues dv;
for (final String fieldName : fieldNames) { for (FieldInfo info : fieldInfos) {
FieldInfo info = infos.fieldInfo(fieldName); if (reader.hasNorms(info.name)) {
if (reader.hasNorms(fieldName)) { dv = reader.normValues(info.name);
dv = reader.normValues(fieldName);
assert dv != null; assert dv != null;
if (dv.getSource().hasArray()) { if (dv.getSource().hasArray()) {
Object array = dv.getSource().getArray(); Object array = dv.getSource().getArray();
if (Array.getLength(array) != reader.maxDoc()) { if (Array.getLength(array) != reader.maxDoc()) {
throw new RuntimeException("norms for field: " + fieldName + " are of the wrong size"); throw new RuntimeException("norms for field: " + info.name + " are of the wrong size");
} }
} }
if (!info.isIndexed || info.omitNorms) { if (!info.isIndexed || info.omitNorms) {
throw new RuntimeException("field: " + fieldName + " should omit norms but has them!"); throw new RuntimeException("field: " + info.name + " should omit norms but has them!");
} }
++status.totFields; ++status.totFields;
} else { } else {
if (reader.normValues(fieldName) != null) { if (reader.normValues(info.name) != null) {
throw new RuntimeException("field: " + fieldName + " should omit norms but has them!"); throw new RuntimeException("field: " + info.name + " should omit norms but has them!");
} }
if (info.isIndexed && !info.omitNorms) { if (info.isIndexed && !info.omitNorms) {
throw new RuntimeException("field: " + fieldName + " should have norms but omits them!"); throw new RuntimeException("field: " + info.name + " should have norms but omits them!");
} }
} }
} }

30
lucene/src/java/org/apache/lucene/index/FieldInfo.java
View File

@ -1,6 +1,5 @@
package org.apache.lucene.index; package org.apache.lucene.index;
import org.apache.lucene.index.DocValues;
/** /**
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
@ -27,15 +26,11 @@ public final class FieldInfo {
public boolean isIndexed; public boolean isIndexed;
private DocValues.Type docValues; private DocValues.Type docValues;
// True if any document indexed term vectors
// true if term vector for this field should be stored
public boolean storeTermVector; public boolean storeTermVector;
public boolean storeOffsetWithTermVector;
public boolean storePositionWithTermVector;
public boolean omitNorms; // omit norms associated with indexed fields public boolean omitNorms; // omit norms associated with indexed fields
public IndexOptions indexOptions; public IndexOptions indexOptions;
public boolean storePayloads; // whether this field stores payloads together with term positions public boolean storePayloads; // whether this field stores payloads together with term positions
/** /**
@ -56,7 +51,6 @@ public final class FieldInfo {
* @lucene.experimental * @lucene.experimental
*/ */
public FieldInfo(String name, boolean isIndexed, int number, boolean storeTermVector, public FieldInfo(String name, boolean isIndexed, int number, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) { boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) {
this.name = name; this.name = name;
this.isIndexed = isIndexed; this.isIndexed = isIndexed;
@ -64,15 +58,11 @@ public final class FieldInfo {
this.docValues = docValues; this.docValues = docValues;
if (isIndexed) { if (isIndexed) {
this.storeTermVector = storeTermVector; this.storeTermVector = storeTermVector;
this.storeOffsetWithTermVector = storeOffsetWithTermVector;
this.storePositionWithTermVector = storePositionWithTermVector;
this.storePayloads = storePayloads; this.storePayloads = storePayloads;
this.omitNorms = omitNorms; this.omitNorms = omitNorms;
this.indexOptions = indexOptions; this.indexOptions = indexOptions;
} else { // for non-indexed fields, leave defaults } else { // for non-indexed fields, leave defaults
this.storeTermVector = false; this.storeTermVector = false;
this.storeOffsetWithTermVector = false;
this.storePositionWithTermVector = false;
this.storePayloads = false; this.storePayloads = false;
this.omitNorms = false; this.omitNorms = false;
this.indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; this.indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
@ -82,14 +72,12 @@ public final class FieldInfo {
@Override @Override
public Object clone() { public Object clone() {
FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, return new FieldInfo(name, isIndexed, number, storeTermVector,
storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues); omitNorms, storePayloads, indexOptions, docValues);
return clone;
} }
// should only be called by FieldInfos#addOrUpdate // should only be called by FieldInfos#addOrUpdate
void update(boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector, void update(boolean isIndexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
if (this.isIndexed != isIndexed) { if (this.isIndexed != isIndexed) {
this.isIndexed = true; // once indexed, always index this.isIndexed = true; // once indexed, always index
@ -98,12 +86,6 @@ public final class FieldInfo {
if (this.storeTermVector != storeTermVector) { if (this.storeTermVector != storeTermVector) {
this.storeTermVector = true; // once vector, always vector this.storeTermVector = true; // once vector, always vector
} }
if (this.storePositionWithTermVector != storePositionWithTermVector) {
this.storePositionWithTermVector = true; // once vector, always vector
}
if (this.storeOffsetWithTermVector != storeOffsetWithTermVector) {
this.storeOffsetWithTermVector = true; // once vector, always vector
}
if (this.storePayloads != storePayloads) { if (this.storePayloads != storePayloads) {
this.storePayloads = true; this.storePayloads = true;
} }
@ -139,9 +121,7 @@ public final class FieldInfo {
return docValues; return docValues;
} }
public void setStoreTermVectors(boolean withPositions, boolean withOffsets) { public void setStoreTermVectors() {
storeTermVector = true; storeTermVector = true;
storePositionWithTermVector |= withPositions;
storeOffsetWithTermVector |= withOffsets;
} }
} }

76
lucene/src/java/org/apache/lucene/index/FieldInfos.java
View File

@ -25,7 +25,6 @@ import java.util.SortedMap;
import java.util.TreeMap; import java.util.TreeMap;
import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.DocValues;
/** Access to the Field Info file that describes document fields and whether or /** Access to the Field Info file that describes document fields and whether or
* not they are indexed. Each segment has a separate Field Info file. Objects * not they are indexed. Each segment has a separate Field Info file. Objects
@ -122,6 +121,16 @@ public final class FieldInfos implements Iterable<FieldInfo> {
} }
} }
public FieldInfos() {
this(new FieldNumberBiMap());
}
public void add(FieldInfos other) {
for(FieldInfo fieldInfo : other){
add(fieldInfo);
}
}
/** /**
* Creates a new FieldInfos instance with the given {@link FieldNumberBiMap}. * Creates a new FieldInfos instance with the given {@link FieldNumberBiMap}.
* If the {@link FieldNumberBiMap} is <code>null</code> this instance will be read-only. * If the {@link FieldNumberBiMap} is <code>null</code> this instance will be read-only.
@ -202,13 +211,10 @@ public final class FieldInfos implements Iterable<FieldInfo> {
* *
* @param names The names of the fields * @param names The names of the fields
* @param storeTermVectors Whether the fields store term vectors or not * @param storeTermVectors Whether the fields store term vectors or not
* @param storePositionWithTermVector true if positions should be stored.
* @param storeOffsetWithTermVector true if offsets should be stored
*/ */
synchronized public void addOrUpdateIndexed(Collection<String> names, boolean storeTermVectors, boolean storePositionWithTermVector, synchronized public void addOrUpdateIndexed(Collection<String> names, boolean storeTermVectors) {
boolean storeOffsetWithTermVector) {
for (String name : names) { for (String name : names) {
addOrUpdate(name, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector); addOrUpdate(name, true, storeTermVectors);
} }
} }
@ -231,21 +237,23 @@ public final class FieldInfos implements Iterable<FieldInfo> {
* *
* @param name The name of the IndexableField * @param name The name of the IndexableField
* @param isIndexed true if the field is indexed * @param isIndexed true if the field is indexed
* @see #addOrUpdate(String, boolean, boolean, boolean, boolean) * @see #addOrUpdate(String, boolean, boolean)
*/ */
synchronized public void addOrUpdate(String name, boolean isIndexed) { synchronized public void addOrUpdate(String name, boolean isIndexed) {
addOrUpdate(name, isIndexed, false, false, false, false); addOrUpdate(name, isIndexed, false, false);
} }
/** /** If the field is not yet known, adds it. If it is known, checks to make
* Calls 5 parameter add with false for term vector positions and offsets. * sure that the isIndexed flag is the same as was given previously for this
* field. If not - marks it as being indexed. Same goes for the TermVector
* parameters.
* *
* @param name The name of the field * @param name The name of the field
* @param isIndexed true if the field is indexed * @param isIndexed true if the field is indexed
* @param storeTermVector true if the term vector should be stored * @param storeTermVector true if the term vector should be stored
*/ */
synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector) { synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector) {
addOrUpdate(name, isIndexed, storeTermVector, false, false, false); addOrUpdate(name, isIndexed, storeTermVector, false);
} }
/** If the field is not yet known, adds it. If it is known, checks to make /** If the field is not yet known, adds it. If it is known, checks to make
@ -256,31 +264,11 @@ public final class FieldInfos implements Iterable<FieldInfo> {
* @param name The name of the field * @param name The name of the field
* @param isIndexed true if the field is indexed * @param isIndexed true if the field is indexed
* @param storeTermVector true if the term vector should be stored * @param storeTermVector true if the term vector should be stored
* @param storePositionWithTermVector true if the term vector with positions should be stored
* @param storeOffsetWithTermVector true if the term vector with offsets should be stored
*/
synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector) {
addOrUpdate(name, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, false);
}
/** If the field is not yet known, adds it. If it is known, checks to make
* sure that the isIndexed flag is the same as was given previously for this
* field. If not - marks it as being indexed. Same goes for the TermVector
* parameters.
*
* @param name The name of the field
* @param isIndexed true if the field is indexed
* @param storeTermVector true if the term vector should be stored
* @param storePositionWithTermVector true if the term vector with positions should be stored
* @param storeOffsetWithTermVector true if the term vector with offsets should be stored
* @param omitNorms true if the norms for the indexed field should be omitted * @param omitNorms true if the norms for the indexed field should be omitted
*/ */
synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector, synchronized public void addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean omitNorms) { boolean omitNorms) {
addOrUpdate(name, isIndexed, storeTermVector, storePositionWithTermVector, addOrUpdate(name, isIndexed, storeTermVector, omitNorms, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null);
storeOffsetWithTermVector, omitNorms, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null);
} }
/** If the field is not yet known, adds it. If it is known, checks to make /** If the field is not yet known, adds it. If it is known, checks to make
@ -291,17 +279,13 @@ public final class FieldInfos implements Iterable<FieldInfo> {
* @param name The name of the field * @param name The name of the field
* @param isIndexed true if the field is indexed * @param isIndexed true if the field is indexed
* @param storeTermVector true if the term vector should be stored * @param storeTermVector true if the term vector should be stored
* @param storePositionWithTermVector true if the term vector with positions should be stored
* @param storeOffsetWithTermVector true if the term vector with offsets should be stored
* @param omitNorms true if the norms for the indexed field should be omitted * @param omitNorms true if the norms for the indexed field should be omitted
* @param storePayloads true if payloads should be stored for this field * @param storePayloads true if payloads should be stored for this field
* @param indexOptions if term freqs should be omitted for this field * @param indexOptions if term freqs should be omitted for this field
*/ */
synchronized public FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector, synchronized public FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) { boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) {
return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, storePositionWithTermVector, return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues);
storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues);
} }
// NOTE: this method does not carry over termVector // NOTE: this method does not carry over termVector
@ -315,13 +299,13 @@ public final class FieldInfos implements Iterable<FieldInfo> {
// rather, each component in the chain should update // rather, each component in the chain should update
// what it "owns". EG fieldType.indexOptions() should // what it "owns". EG fieldType.indexOptions() should
// be updated by maybe FreqProxTermsWriterPerField: // be updated by maybe FreqProxTermsWriterPerField:
return addOrUpdateInternal(name, -1, fieldType.indexed(), false, false, false, return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
fieldType.omitNorms(), false, fieldType.omitNorms(), false,
fieldType.indexOptions(), null); fieldType.indexOptions(), null);
} }
synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed, synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) { boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) {
if (globalFieldNumbers == null) { if (globalFieldNumbers == null) {
throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos"); throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos");
@ -329,9 +313,9 @@ public final class FieldInfos implements Iterable<FieldInfo> {
FieldInfo fi = fieldInfo(name); FieldInfo fi = fieldInfo(name);
if (fi == null) { if (fi == null) {
final int fieldNumber = nextFieldNumber(name, preferredFieldNumber); final int fieldNumber = nextFieldNumber(name, preferredFieldNumber);
fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues); fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions, docValues);
} else { } else {
fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions); fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions);
fi.setDocValuesType(docValues); fi.setDocValuesType(docValues);
} }
version++; version++;
@ -341,7 +325,6 @@ public final class FieldInfos implements Iterable<FieldInfo> {
synchronized public FieldInfo add(FieldInfo fi) { synchronized public FieldInfo add(FieldInfo fi) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments // IMPORTANT - reuse the field number if possible for consistent field numbers across segments
return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed, fi.storeTermVector, return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed, fi.storeTermVector,
fi.storePositionWithTermVector, fi.storeOffsetWithTermVector,
fi.omitNorms, fi.storePayloads, fi.omitNorms, fi.storePayloads,
fi.indexOptions, fi.getDocValuesType()); fi.indexOptions, fi.getDocValuesType());
} }
@ -350,14 +333,13 @@ public final class FieldInfos implements Iterable<FieldInfo> {
* NOTE: if you call this method from a public method make sure you check if we are modifiable and throw an exception otherwise * NOTE: if you call this method from a public method make sure you check if we are modifiable and throw an exception otherwise
*/ */
private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed, private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed,
boolean storeTermVector, boolean storePositionWithTermVector, boolean storeTermVector, boolean omitNorms, boolean storePayloads,
boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValuesType) { IndexOptions indexOptions, DocValues.Type docValuesType) {
// don't check modifiable here since we use that to initially build up FIs // don't check modifiable here since we use that to initially build up FIs
if (globalFieldNumbers != null) { if (globalFieldNumbers != null) {
globalFieldNumbers.setIfNotSet(fieldNumber, name); globalFieldNumbers.setIfNotSet(fieldNumber, name);
} }
final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, storePositionWithTermVector, final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType);
storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType);
putInternal(fi); putInternal(fi);
return fi; return fi;
} }

12
lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
View File

@ -22,7 +22,6 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.Map; import java.util.Map;
import java.util.Comparator; import java.util.Comparator;
@ -294,6 +293,11 @@ public class FilterIndexReader extends IndexReader {
return in.getLiveDocs(); return in.getLiveDocs();
} }
@Override
public FieldInfos getFieldInfos() {
return in.getFieldInfos();
}
@Override @Override
public Fields getTermVectors(int docID) public Fields getTermVectors(int docID)
throws IOException { throws IOException {
@ -342,12 +346,6 @@ public class FilterIndexReader extends IndexReader {
in.close(); in.close();
} }
@Override
public Collection<String> getFieldNames(IndexReader.FieldOption fieldNames) {
ensureOpen();
return in.getFieldNames(fieldNames);
}
@Override @Override
public long getVersion() { public long getVersion() {
ensureOpen(); ensureOpen();

48
lucene/src/java/org/apache/lucene/index/IndexReader.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
@ -29,7 +28,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor; import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.search.SearcherManager; // javadocs import org.apache.lucene.search.SearcherManager; // javadocs
import org.apache.lucene.store.*; import org.apache.lucene.store.*;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
@ -108,39 +106,6 @@ public abstract class IndexReader implements Closeable {
} }
} }
/**
* Constants describing field properties, for example used for
* {@link IndexReader#getFieldNames(FieldOption)}.
*/
public static enum FieldOption {
/** All fields */
ALL,
/** All indexed fields */
INDEXED,
/** All fields that store payloads */
STORES_PAYLOADS,
/** All fields that omit tf */
OMIT_TERM_FREQ_AND_POSITIONS,
/** All fields that omit positions */
OMIT_POSITIONS,
/** All fields which are not indexed */
UNINDEXED,
/** All fields which are indexed with termvectors enabled */
INDEXED_WITH_TERMVECTOR,
/** All fields which are indexed but don't have termvectors enabled */
INDEXED_NO_TERMVECTOR,
/** All fields with termvectors enabled. Please note that only standard termvector fields are returned */
TERMVECTOR,
/** All fields with termvectors with position values enabled */
TERMVECTOR_WITH_POSITION,
/** All fields with termvectors with offset values enabled */
TERMVECTOR_WITH_OFFSET,
/** All fields with termvectors with offset values and position values enabled */
TERMVECTOR_WITH_POSITION_OFFSET,
/** All fields holding doc values */
DOC_VALUES
}
private volatile boolean closed; private volatile boolean closed;
private final AtomicInteger refCount = new AtomicInteger(); private final AtomicInteger refCount = new AtomicInteger();
@ -897,15 +862,14 @@ public abstract class IndexReader implements Closeable {
/** Implements close. */ /** Implements close. */
protected abstract void doClose() throws IOException; protected abstract void doClose() throws IOException;
/** /**
* Get a list of unique field names that exist in this index and have the specified * Get the {@link FieldInfos} describing all fields in
* field option information. * this reader. NOTE: do not make any changes to the
* @param fldOption specifies which field option should be available for the returned fields * returned FieldInfos!
* @return Collection of Strings indicating the names of the fields. *
* @see IndexReader.FieldOption * @lucene.experimental
*/ */
public abstract Collection<String> getFieldNames(FieldOption fldOption); public abstract FieldInfos getFieldInfos();
/** Returns the {@link Bits} representing live (not /** Returns the {@link Bits} representing live (not
* deleted) docs. A set bit indicates the doc ID has not * deleted) docs. A set bit indicates the doc ID has not

38
lucene/src/java/org/apache/lucene/index/ParallelReader.java
View File

@ -17,12 +17,13 @@ package org.apache.lucene.index;
* limitations under the License. * limitations under the License.
*/ */
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
/** An IndexReader which reads multiple, parallel indexes. Each index added /** An IndexReader which reads multiple, parallel indexes. Each index added
* must have the same number of documents, but typically each contains * must have the same number of documents, but typically each contains
@ -53,6 +54,7 @@ public class ParallelReader extends IndexReader {
private int maxDoc; private int maxDoc;
private int numDocs; private int numDocs;
private boolean hasDeletions; private boolean hasDeletions;
private final FieldInfos fieldInfos;
private final ParallelFields fields = new ParallelFields(); private final ParallelFields fields = new ParallelFields();
@ -68,6 +70,7 @@ public class ParallelReader extends IndexReader {
public ParallelReader(boolean closeSubReaders) throws IOException { public ParallelReader(boolean closeSubReaders) throws IOException {
super(); super();
this.incRefReaders = !closeSubReaders; this.incRefReaders = !closeSubReaders;
fieldInfos = new FieldInfos();
} }
/** {@inheritDoc} */ /** {@inheritDoc} */
@ -120,12 +123,13 @@ public class ParallelReader extends IndexReader {
throw new IllegalArgumentException throw new IllegalArgumentException
("All readers must have same numDocs: "+numDocs+"!="+reader.numDocs()); ("All readers must have same numDocs: "+numDocs+"!="+reader.numDocs());
Collection<String> fields = reader.getFieldNames(IndexReader.FieldOption.ALL); final FieldInfos readerFieldInfos = ReaderUtil.getMergedFieldInfos(reader);
readerToFields.put(reader, fields); for(FieldInfo fieldInfo : readerFieldInfos) { // update fieldToReader map
for (final String field : fields) { // update fieldToReader map // NOTE: first reader having a given field "wins":
if (fieldToReader.get(field) == null) { if (fieldToReader.get(fieldInfo.name) == null) {
fieldToReader.put(field, reader); fieldInfos.add(fieldInfo);
this.fields.addField(field, MultiFields.getFields(reader).terms(field)); fieldToReader.put(fieldInfo.name, reader);
this.fields.addField(fieldInfo.name, MultiFields.getFields(reader).terms(fieldInfo.name));
} }
} }
@ -193,6 +197,11 @@ public class ParallelReader extends IndexReader {
} }
} }
@Override
public FieldInfos getFieldInfos() {
return fieldInfos;
}
@Override @Override
public Bits getLiveDocs() { public Bits getLiveDocs() {
ensureOpen(); ensureOpen();
@ -383,17 +392,6 @@ public class ParallelReader extends IndexReader {
} }
} }
@Override
public Collection<String> getFieldNames (IndexReader.FieldOption fieldNames) {
ensureOpen();
Set<String> fieldSet = new HashSet<String>();
for (final IndexReader reader : readers) {
Collection<String> names = reader.getFieldNames(fieldNames);
fieldSet.addAll(names);
}
return fieldSet;
}
@Override @Override
public ReaderContext getTopReaderContext() { public ReaderContext getTopReaderContext() {
ensureOpen(); ensureOpen();

47
lucene/src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -30,9 +29,6 @@ import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.PerDocConsumer; import org.apache.lucene.codecs.PerDocConsumer;
import org.apache.lucene.codecs.StoredFieldsWriter; import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.TermVectorsWriter; import org.apache.lucene.codecs.TermVectorsWriter;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
@ -135,18 +131,6 @@ final class SegmentMerger {
return mergeState; return mergeState;
} }
private static void addIndexed(IndexReader reader, FieldInfos fInfos,
Collection<String> names, boolean storeTermVectors,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
boolean storePayloads, IndexOptions indexOptions)
throws IOException {
for (String field : names) {
fInfos.addOrUpdate(field, true, storeTermVectors,
storePositionWithTermVector, storeOffsetWithTermVector, !reader
.hasNorms(field), storePayloads, indexOptions, null);
}
}
private void setMatchingSegmentReaders() { private void setMatchingSegmentReaders() {
// If the i'th reader is a SegmentReader and has // If the i'th reader is a SegmentReader and has
// identical fieldName -> number mapping, then this // identical fieldName -> number mapping, then this
@ -160,10 +144,15 @@ final class SegmentMerger {
// stored fields: // stored fields:
for (int i = 0; i < numReaders; i++) { for (int i = 0; i < numReaders; i++) {
MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i); MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i);
// TODO: we may be able to broaden this to
// non-SegmentReaders, since FieldInfos is now
// required? But... this'd also require exposing
// bulk-copy (TVs and stored fields) API in foreign
// readers..
if (reader.reader instanceof SegmentReader) { if (reader.reader instanceof SegmentReader) {
SegmentReader segmentReader = (SegmentReader) reader.reader; SegmentReader segmentReader = (SegmentReader) reader.reader;
boolean same = true; boolean same = true;
FieldInfos segmentFieldInfos = segmentReader.fieldInfos(); FieldInfos segmentFieldInfos = segmentReader.getFieldInfos();
for (FieldInfo fi : segmentFieldInfos) { for (FieldInfo fi : segmentFieldInfos) {
if (!mergeState.fieldInfos.fieldName(fi.number).equals(fi.name)) { if (!mergeState.fieldInfos.fieldName(fi.number).equals(fi.name)) {
same = false; same = false;
@ -208,9 +197,7 @@ final class SegmentMerger {
for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) { for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) {
final IndexReader reader = readerAndLiveDocs.reader; final IndexReader reader = readerAndLiveDocs.reader;
if (reader instanceof SegmentReader) { FieldInfos readerFieldInfos = reader.getFieldInfos();
SegmentReader segmentReader = (SegmentReader) reader;
FieldInfos readerFieldInfos = segmentReader.fieldInfos();
for (FieldInfo fi : readerFieldInfos) { for (FieldInfo fi : readerFieldInfos) {
FieldInfo merged = mergeState.fieldInfos.add(fi); FieldInfo merged = mergeState.fieldInfos.add(fi);
// update the type promotion mapping for this reader // update the type promotion mapping for this reader
@ -219,26 +206,6 @@ final class SegmentMerger {
docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name))); docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
} }
} }
} else {
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR), true, false, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.OMIT_POSITIONS), false, false, false, false, IndexOptions.DOCS_AND_FREQS);
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.OMIT_TERM_FREQ_AND_POSITIONS), false, false, false, false, IndexOptions.DOCS_ONLY);
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.INDEXED), false, false, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
mergeState.fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.UNINDEXED), false);
Collection<String> dvNames = reader.getFieldNames(FieldOption.DOC_VALUES);
mergeState.fieldInfos.addOrUpdate(dvNames, false);
for (String dvName : dvNames) {
FieldInfo merged = mergeState.fieldInfos.fieldInfo(dvName);
DocValues docValues = reader.docValues(dvName);
merged.setDocValuesType(docValues.type());
TypePromoter previous = docValuesTypes.get(merged);
docValuesTypes.put(merged, mergeDocValuesType(previous, docValues));
}
}
} }
// update any promoted doc values types: // update any promoted doc values types:

64
lucene/src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -18,16 +18,11 @@ package org.apache.lucene.index;
*/ */
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.codecs.PerDocProducer; import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.StoredFieldsReader; import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader; import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.FieldCache; // javadocs import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector; import org.apache.lucene.util.BitVector;
@ -162,7 +157,8 @@ public final class SegmentReader extends IndexReader {
return liveDocs != null; return liveDocs != null;
} }
FieldInfos fieldInfos() { @Override
public FieldInfos getFieldInfos() {
return core.fieldInfos; return core.fieldInfos;
} }
@ -198,62 +194,6 @@ public final class SegmentReader extends IndexReader {
return si.docCount; return si.docCount;
} }
/**
* @see IndexReader#getFieldNames(org.apache.lucene.index.IndexReader.FieldOption)
*/
@Override
public Collection<String> getFieldNames(IndexReader.FieldOption fieldOption) {
ensureOpen();
Set<String> fieldSet = new HashSet<String>();
for (FieldInfo fi : core.fieldInfos) {
if (fieldOption == IndexReader.FieldOption.ALL) {
fieldSet.add(fi.name);
}
else if (!fi.isIndexed && fieldOption == IndexReader.FieldOption.UNINDEXED) {
fieldSet.add(fi.name);
}
else if (fi.indexOptions == IndexOptions.DOCS_ONLY && fieldOption == IndexReader.FieldOption.OMIT_TERM_FREQ_AND_POSITIONS) {
fieldSet.add(fi.name);
}
else if (fi.indexOptions == IndexOptions.DOCS_AND_FREQS && fieldOption == IndexReader.FieldOption.OMIT_POSITIONS) {
fieldSet.add(fi.name);
}
else if (fi.storePayloads && fieldOption == IndexReader.FieldOption.STORES_PAYLOADS) {
fieldSet.add(fi.name);
}
else if (fi.isIndexed && fieldOption == IndexReader.FieldOption.INDEXED) {
fieldSet.add(fi.name);
}
else if (fi.isIndexed && fi.storeTermVector == false && fieldOption == IndexReader.FieldOption.INDEXED_NO_TERMVECTOR) {
fieldSet.add(fi.name);
}
else if (fi.storeTermVector == true &&
fi.storePositionWithTermVector == false &&
fi.storeOffsetWithTermVector == false &&
fieldOption == IndexReader.FieldOption.TERMVECTOR) {
fieldSet.add(fi.name);
}
else if (fi.isIndexed && fi.storeTermVector && fieldOption == IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR) {
fieldSet.add(fi.name);
}
else if (fi.storePositionWithTermVector && fi.storeOffsetWithTermVector == false && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION) {
fieldSet.add(fi.name);
}
else if (fi.storeOffsetWithTermVector && fi.storePositionWithTermVector == false && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET) {
fieldSet.add(fi.name);
}
else if ((fi.storeOffsetWithTermVector && fi.storePositionWithTermVector) &&
fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET) {
fieldSet.add(fi.name);
}
else if (fi.hasDocValues() && fieldOption == IndexReader.FieldOption.DOC_VALUES) {
fieldSet.add(fi.name);
}
}
return fieldSet;
}
@Override @Override
public boolean hasNorms(String field) { public boolean hasNorms(String field) {
ensureOpen(); ensureOpen();

5
lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
View File

@ -103,4 +103,9 @@ public final class SlowMultiReaderWrapper extends FilterIndexReader {
ensureOpen(); ensureOpen();
return readerContext; return readerContext;
} }
@Override
public FieldInfos getFieldInfos() {
return ReaderUtil.getMergedFieldInfos(in);
}
} }

2
lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
View File

@ -151,7 +151,7 @@ final class TermVectorsConsumerPerField extends TermsHashConsumerPerField {
termsHashPerField.reset(); termsHashPerField.reset();
// commit the termVectors once successful success - FI will otherwise reset them // commit the termVectors once successful success - FI will otherwise reset them
fieldInfo.setStoreTermVectors(doVectorPositions, doVectorOffsets); fieldInfo.setStoreTermVectors();
} }
void shrinkHash() { void shrinkHash() {

26
lucene/src/java/org/apache/lucene/util/ReaderUtil.java
View File

@ -18,9 +18,13 @@ package org.apache.lucene.util;
*/ */
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.io.IOException; import java.io.IOException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.CompositeReaderContext; import org.apache.lucene.index.IndexReader.CompositeReaderContext;
@ -292,4 +296,26 @@ public final class ReaderUtil {
} }
return hi; return hi;
} }
public static Collection<String> getIndexedFields(IndexReader reader) {
final Collection<String> fields = new HashSet<String>();
for(FieldInfo fieldInfo : getMergedFieldInfos(reader)) {
if (fieldInfo.isIndexed) {
fields.add(fieldInfo.name);
}
}
return fields;
}
/** Call this to get the (merged) FieldInfos for a
* composite reader */
public static FieldInfos getMergedFieldInfos(IndexReader reader) {
final List<IndexReader> subReaders = new ArrayList<IndexReader>();
ReaderUtil.gatherSubReaders(subReaders, reader);
final FieldInfos fieldInfos = new FieldInfos();
for(IndexReader subReader : subReaders) {
fieldInfos.add(subReader.getFieldInfos());
}
return fieldInfos;
}
} }

18
lucene/src/site/build/site/fileformats.html
View File

@ -1740,10 +1740,6 @@ document.write("Last Published: " + document.lastModified);
without term vectors. without term vectors.
</li> </li>
<li>If the third lowest-order bit is set (0x04), term positions are stored with the term vectors.</li>
<li>If the fourth lowest-order bit is set (0x08), term offsets are stored with the term vectors.</li>
<li>If the fifth lowest-order bit is set (0x10), norms are omitted for the indexed field.</li> <li>If the fifth lowest-order bit is set (0x10), norms are omitted for the indexed field.</li>
<li>If the sixth lowest-order bit is set (0x20), payloads are stored for the indexed field.</li> <li>If the sixth lowest-order bit is set (0x20), payloads are stored for the indexed field.</li>
@ -1908,7 +1904,7 @@ document.write("Last Published: " + document.lastModified);
</li> </li>
</ol> </ol>
<a name="N106F0"></a><a name="Term Dictionary"></a> <a name="N106EA"></a><a name="Term Dictionary"></a>
<h3 class="boxed">Term Dictionary</h3> <h3 class="boxed">Term Dictionary</h3>
<p> <p>
The term dictionary is represented as two files: The term dictionary is represented as two files:
@ -2100,7 +2096,7 @@ document.write("Last Published: " + document.lastModified);
</li> </li>
</ol> </ol>
<a name="N10774"></a><a name="Frequencies"></a> <a name="N1076E"></a><a name="Frequencies"></a>
<h3 class="boxed">Frequencies</h3> <h3 class="boxed">Frequencies</h3>
<p> <p>
The .frq file contains the lists of documents The .frq file contains the lists of documents
@ -2228,7 +2224,7 @@ document.write("Last Published: " + document.lastModified);
entry in level-1. In the example has entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a pointer entry in level-1. In the example has entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a pointer
to entry 31 on level 0. to entry 31 on level 0.
</p> </p>
<a name="N107FC"></a><a name="Positions"></a> <a name="N107F6"></a><a name="Positions"></a>
<h3 class="boxed">Positions</h3> <h3 class="boxed">Positions</h3>
<p> <p>
The .prx file contains the lists of positions that The .prx file contains the lists of positions that
@ -2298,7 +2294,7 @@ document.write("Last Published: " + document.lastModified);
Payload. If PayloadLength is not stored, then this Payload has the same Payload. If PayloadLength is not stored, then this Payload has the same
length as the Payload at the previous position. length as the Payload at the previous position.
</p> </p>
<a name="N10838"></a><a name="Normalization Factors"></a> <a name="N10832"></a><a name="Normalization Factors"></a>
<h3 class="boxed">Normalization Factors</h3> <h3 class="boxed">Normalization Factors</h3>
<p>There's a single .nrm file containing all norms: <p>There's a single .nrm file containing all norms:
</p> </p>
@ -2378,7 +2374,7 @@ document.write("Last Published: " + document.lastModified);
</p> </p>
<p>Separate norm files are created (when adequate) for both compound and non compound segments. <p>Separate norm files are created (when adequate) for both compound and non compound segments.
</p> </p>
<a name="N10889"></a><a name="Term Vectors"></a> <a name="N10883"></a><a name="Term Vectors"></a>
<h3 class="boxed">Term Vectors</h3> <h3 class="boxed">Term Vectors</h3>
<p> <p>
Term Vector support is an optional on a field by Term Vector support is an optional on a field by
@ -2514,7 +2510,7 @@ document.write("Last Published: " + document.lastModified);
</li> </li>
</ol> </ol>
<a name="N10925"></a><a name="Deleted Documents"></a> <a name="N1091F"></a><a name="Deleted Documents"></a>
<h3 class="boxed">Deleted Documents</h3> <h3 class="boxed">Deleted Documents</h3>
<p>The .del file is <p>The .del file is
optional, and only exists when a segment contains deletions. optional, and only exists when a segment contains deletions.
@ -2578,7 +2574,7 @@ document.write("Last Published: " + document.lastModified);
</div> </div>
<a name="N1095F"></a><a name="Limitations"></a> <a name="N10959"></a><a name="Limitations"></a>
<h2 class="boxed">Limitations</h2> <h2 class="boxed">Limitations</h2>
<div class="section"> <div class="section">
<p> <p>

2
lucene/src/site/src/documentation/content/xdocs/fileformats.xml
View File

@ -1216,8 +1216,6 @@
bit is one for fields that have term vectors stored, and zero for fields bit is one for fields that have term vectors stored, and zero for fields
without term vectors. without term vectors.
</li> </li>
<li>If the third lowest-order bit is set (0x04), term positions are stored with the term vectors.</li>
<li>If the fourth lowest-order bit is set (0x08), term offsets are stored with the term vectors.</li>
<li>If the fifth lowest-order bit is set (0x10), norms are omitted for the indexed field.</li> <li>If the fifth lowest-order bit is set (0x10), norms are omitted for the indexed field.</li>
<li>If the sixth lowest-order bit is set (0x20), payloads are stored for the indexed field.</li> <li>If the sixth lowest-order bit is set (0x20), payloads are stored for the indexed field.</li>
<li>If the seventh lowest-order bit is set (0x40), term frequencies and positions omitted for the indexed field.</li> <li>If the seventh lowest-order bit is set (0x40), term frequencies and positions omitted for the indexed field.</li>

2
lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
View File

@ -296,8 +296,6 @@ public class TestConsistentFieldNumbers extends LuceneTestCase {
Field expected = getField(Integer.parseInt(fi.name)); Field expected = getField(Integer.parseInt(fi.name));
assertEquals(expected.fieldType().indexed(), fi.isIndexed); assertEquals(expected.fieldType().indexed(), fi.isIndexed);
assertEquals(expected.fieldType().storeTermVectors(), fi.storeTermVector); assertEquals(expected.fieldType().storeTermVectors(), fi.storeTermVector);
assertEquals(expected.fieldType().storeTermVectorPositions(), fi.storePositionWithTermVector);
assertEquals(expected.fieldType().storeTermVectorOffsets(), fi.storeOffsetWithTermVector);
} }
} }

4
lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
View File

@ -94,7 +94,7 @@ public class TestDocumentWriter extends LuceneTestCase {
// test that the norms are not present in the segment if // test that the norms are not present in the segment if
// omitNorms is true // omitNorms is true
for (FieldInfo fi : reader.fieldInfos()) { for (FieldInfo fi : reader.getFieldInfos()) {
if (fi.isIndexed) { if (fi.isIndexed) {
assertTrue(fi.omitNorms == !reader.hasNorms(fi.name)); assertTrue(fi.omitNorms == !reader.hasNorms(fi.name));
} }
@ -327,7 +327,7 @@ public class TestDocumentWriter extends LuceneTestCase {
_TestUtil.checkIndex(dir); _TestUtil.checkIndex(dir);
SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
// f1 // f1
assertFalse("f1 should have no norms", reader.hasNorms("f1")); assertFalse("f1 should have no norms", reader.hasNorms("f1"));
assertEquals("omitTermFreqAndPositions field bit should not be set for f1", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions); assertEquals("omitTermFreqAndPositions field bit should not be set for f1", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions);

17
lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
View File

@ -27,7 +27,6 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
@ -35,6 +34,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.automaton.AutomatonTestUtil; import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp; import org.apache.lucene.util.automaton.RegExp;
@ -517,12 +517,23 @@ public class TestDuelingCodecs extends LuceneTestCase {
} }
} }
private static Set<String> getDVFields(IndexReader reader) {
Set<String> fields = new HashSet<String>();
for(FieldInfo fi : ReaderUtil.getMergedFieldInfos(reader)) {
if (fi.hasDocValues()) {
fields.add(fi.name);
}
}
return fields;
}
/** /**
* checks that docvalues across all fields are equivalent * checks that docvalues across all fields are equivalent
*/ */
public void assertDocValues(IndexReader leftReader, IndexReader rightReader) throws Exception { public void assertDocValues(IndexReader leftReader, IndexReader rightReader) throws Exception {
Set<String> leftValues = new HashSet<String>(leftReader.getFieldNames(FieldOption.DOC_VALUES)); Set<String> leftValues = getDVFields(leftReader);
Set<String> rightValues = new HashSet<String>(rightReader.getFieldNames(FieldOption.DOC_VALUES)); Set<String> rightValues = getDVFields(rightReader);
assertEquals(info, leftValues, rightValues); assertEquals(info, leftValues, rightValues);
for (String field : leftValues) { for (String field : leftValues) {

11
lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
View File

@ -133,21 +133,14 @@ public class TestFieldInfos extends LuceneTestCase {
} }
try { try {
readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(), readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
random.nextBoolean(), random.nextBoolean()); random.nextBoolean());
fail("instance should be read only"); fail("instance should be read only");
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
// expected // expected
} }
try { try {
readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(), readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
random.nextBoolean(), random.nextBoolean(), random.nextBoolean()); random.nextBoolean(),
fail("instance should be read only");
} catch (IllegalStateException e) {
// expected
}
try {
readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null); random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null);
fail("instance should be read only"); fail("instance should be read only");
} catch (IllegalStateException e) { } catch (IllegalStateException e) {

6
lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
View File

@ -80,24 +80,18 @@ public class TestFieldsReader extends LuceneTestCase {
assertTrue(field != null); assertTrue(field != null);
assertTrue(field.fieldType().storeTermVectors()); assertTrue(field.fieldType().storeTermVectors());
assertTrue(field.fieldType().storeTermVectorOffsets());
assertTrue(field.fieldType().storeTermVectorPositions());
assertFalse(field.fieldType().omitNorms()); assertFalse(field.fieldType().omitNorms());
assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
field = (Field) doc.getField(DocHelper.TEXT_FIELD_3_KEY); field = (Field) doc.getField(DocHelper.TEXT_FIELD_3_KEY);
assertTrue(field != null); assertTrue(field != null);
assertFalse(field.fieldType().storeTermVectors()); assertFalse(field.fieldType().storeTermVectors());
assertFalse(field.fieldType().storeTermVectorOffsets());
assertFalse(field.fieldType().storeTermVectorPositions());
assertTrue(field.fieldType().omitNorms()); assertTrue(field.fieldType().omitNorms());
assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
field = (Field) doc.getField(DocHelper.NO_TF_KEY); field = (Field) doc.getField(DocHelper.NO_TF_KEY);
assertTrue(field != null); assertTrue(field != null);
assertFalse(field.fieldType().storeTermVectors()); assertFalse(field.fieldType().storeTermVectors());
assertFalse(field.fieldType().storeTermVectorOffsets());
assertFalse(field.fieldType().storeTermVectorPositions());
assertFalse(field.fieldType().omitNorms()); assertFalse(field.fieldType().omitNorms());
assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_ONLY); assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_ONLY);

23
lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java
View File

@ -18,20 +18,20 @@ package org.apache.lucene.index;
*/ */
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Bits;
import java.io.IOException; import java.io.IOException;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.lang.reflect.Modifier; import java.lang.reflect.Modifier;
import java.util.HashSet; import java.util.HashSet;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ReaderUtil;
public class TestFilterIndexReader extends LuceneTestCase { public class TestFilterIndexReader extends LuceneTestCase {
private static class TestReader extends FilterIndexReader { private static class TestReader extends FilterIndexReader {
@ -121,6 +121,11 @@ public class TestFilterIndexReader extends LuceneTestCase {
public Fields fields() throws IOException { public Fields fields() throws IOException {
return new TestFields(super.fields()); return new TestFields(super.fields());
} }
@Override
public FieldInfos getFieldInfos() {
return ReaderUtil.getMergedFieldInfos(in);
}
} }
/** /**

129
lucene/src/test/org/apache/lucene/index/TestIndexReader.java
View File

@ -36,7 +36,6 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField; import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField; import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.FieldCache;
@ -46,6 +45,7 @@ import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util._TestUtil; import org.apache.lucene.util._TestUtil;
import org.junit.Assume; import org.junit.Assume;
@ -102,11 +102,11 @@ public class TestIndexReader extends LuceneTestCase {
writer.close(); writer.close();
// set up reader // set up reader
IndexReader reader = IndexReader.open(d); IndexReader reader = IndexReader.open(d);
Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL); FieldInfos fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
assertTrue(fieldNames.contains("keyword")); assertNotNull(fieldInfos.fieldInfo("keyword"));
assertTrue(fieldNames.contains("text")); assertNotNull(fieldInfos.fieldInfo("text"));
assertTrue(fieldNames.contains("unindexed")); assertNotNull(fieldInfos.fieldInfo("unindexed"));
assertTrue(fieldNames.contains("unstored")); assertNotNull(fieldInfos.fieldInfo("unstored"));
reader.close(); reader.close();
// add more documents // add more documents
writer = new IndexWriter( writer = new IndexWriter(
@ -160,61 +160,66 @@ public class TestIndexReader extends LuceneTestCase {
} }
writer.close(); writer.close();
// verify fields again // verify fields again
reader = IndexReader.open(d); reader = IndexReader.open(d);
fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL); fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
assertEquals(13, fieldNames.size()); // the following fields
assertTrue(fieldNames.contains("keyword")); Collection<String> allFieldNames = new HashSet<String>();
assertTrue(fieldNames.contains("text")); Collection<String> indexedFieldNames = new HashSet<String>();
assertTrue(fieldNames.contains("unindexed")); Collection<String> notIndexedFieldNames = new HashSet<String>();
assertTrue(fieldNames.contains("unstored")); Collection<String> tvFieldNames = new HashSet<String>();
assertTrue(fieldNames.contains("keyword2"));
assertTrue(fieldNames.contains("text2")); for(FieldInfo fieldInfo : fieldInfos) {
assertTrue(fieldNames.contains("unindexed2")); final String name = fieldInfo.name;
assertTrue(fieldNames.contains("unstored2")); allFieldNames.add(name);
assertTrue(fieldNames.contains("tvnot")); if (fieldInfo.isIndexed) {
assertTrue(fieldNames.contains("termvector")); indexedFieldNames.add(name);
assertTrue(fieldNames.contains("tvposition")); } else {
assertTrue(fieldNames.contains("tvoffset")); notIndexedFieldNames.add(name);
assertTrue(fieldNames.contains("tvpositionoffset")); }
if (fieldInfo.storeTermVector) {
tvFieldNames.add(name);
}
}
assertTrue(allFieldNames.contains("keyword"));
assertTrue(allFieldNames.contains("text"));
assertTrue(allFieldNames.contains("unindexed"));
assertTrue(allFieldNames.contains("unstored"));
assertTrue(allFieldNames.contains("keyword2"));
assertTrue(allFieldNames.contains("text2"));
assertTrue(allFieldNames.contains("unindexed2"));
assertTrue(allFieldNames.contains("unstored2"));
assertTrue(allFieldNames.contains("tvnot"));
assertTrue(allFieldNames.contains("termvector"));
assertTrue(allFieldNames.contains("tvposition"));
assertTrue(allFieldNames.contains("tvoffset"));
assertTrue(allFieldNames.contains("tvpositionoffset"));
// verify that only indexed fields were returned // verify that only indexed fields were returned
fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED); assertEquals(11, indexedFieldNames.size()); // 6 original + the 5 termvector fields
assertEquals(11, fieldNames.size()); // 6 original + the 5 termvector fields assertTrue(indexedFieldNames.contains("keyword"));
assertTrue(fieldNames.contains("keyword")); assertTrue(indexedFieldNames.contains("text"));
assertTrue(fieldNames.contains("text")); assertTrue(indexedFieldNames.contains("unstored"));
assertTrue(fieldNames.contains("unstored")); assertTrue(indexedFieldNames.contains("keyword2"));
assertTrue(fieldNames.contains("keyword2")); assertTrue(indexedFieldNames.contains("text2"));
assertTrue(fieldNames.contains("text2")); assertTrue(indexedFieldNames.contains("unstored2"));
assertTrue(fieldNames.contains("unstored2")); assertTrue(indexedFieldNames.contains("tvnot"));
assertTrue(fieldNames.contains("tvnot")); assertTrue(indexedFieldNames.contains("termvector"));
assertTrue(fieldNames.contains("termvector")); assertTrue(indexedFieldNames.contains("tvposition"));
assertTrue(fieldNames.contains("tvposition")); assertTrue(indexedFieldNames.contains("tvoffset"));
assertTrue(fieldNames.contains("tvoffset")); assertTrue(indexedFieldNames.contains("tvpositionoffset"));
assertTrue(fieldNames.contains("tvpositionoffset"));
// verify that only unindexed fields were returned // verify that only unindexed fields were returned
fieldNames = reader.getFieldNames(IndexReader.FieldOption.UNINDEXED); assertEquals(2, notIndexedFieldNames.size()); // the following fields
assertEquals(2, fieldNames.size()); // the following fields assertTrue(notIndexedFieldNames.contains("unindexed"));
assertTrue(fieldNames.contains("unindexed")); assertTrue(notIndexedFieldNames.contains("unindexed2"));
assertTrue(fieldNames.contains("unindexed2"));
// verify index term vector fields // verify index term vector fields
fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR); assertEquals(tvFieldNames.toString(), 4, tvFieldNames.size()); // 4 field has term vector only
assertEquals(1, fieldNames.size()); // 1 field has term vector only assertTrue(tvFieldNames.contains("termvector"));
assertTrue(fieldNames.contains("termvector"));
fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION);
assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
assertTrue(fieldNames.contains("tvposition"));
fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET);
assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
assertTrue(fieldNames.contains("tvoffset"));
fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET);
assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
assertTrue(fieldNames.contains("tvpositionoffset"));
reader.close(); reader.close();
d.close(); d.close();
} }
@ -519,19 +524,19 @@ public class TestIndexReader extends LuceneTestCase {
} }
// check field names // check field names
Collection<String> fields1 = index1.getFieldNames(FieldOption.ALL); FieldInfos fieldInfos1 = ReaderUtil.getMergedFieldInfos(index1);
Collection<String> fields2 = index1.getFieldNames(FieldOption.ALL); FieldInfos fieldInfos2 = ReaderUtil.getMergedFieldInfos(index2);
assertEquals("IndexReaders have different numbers of fields.", fields1.size(), fields2.size()); assertEquals("IndexReaders have different numbers of fields.", fieldInfos1.size(), fieldInfos2.size());
Iterator<String> it1 = fields1.iterator(); final int numFields = fieldInfos1.size();
Iterator<String> it2 = fields1.iterator(); for(int fieldID=0;fieldID<numFields;fieldID++) {
while (it1.hasNext()) { final FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldID);
assertEquals("Different field names.", it1.next(), it2.next()); final FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldID);
assertEquals("Different field names.", fieldInfo1.name, fieldInfo2.name);
} }
// check norms // check norms
it1 = fields1.iterator(); for(FieldInfo fieldInfo : fieldInfos1) {
while (it1.hasNext()) { String curField = fieldInfo.name;
String curField = it1.next();
DocValues norms1 = MultiDocValues.getNormDocValues(index1, curField); DocValues norms1 = MultiDocValues.getNormDocValues(index1, curField);
DocValues norms2 = MultiDocValues.getNormDocValues(index2, curField); DocValues norms2 = MultiDocValues.getNormDocValues(index2, curField);
if (norms1 != null && norms2 != null) if (norms1 != null && norms2 != null)

6
lucene/src/test/org/apache/lucene/index/TestOmitNorms.java
View File

@ -67,7 +67,7 @@ public class TestOmitNorms extends LuceneTestCase {
writer.close(); writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms);
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);
@ -121,7 +121,7 @@ public class TestOmitNorms extends LuceneTestCase {
writer.close(); writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms);
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);
@ -169,7 +169,7 @@ public class TestOmitNorms extends LuceneTestCase {
writer.close(); writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
assertTrue("OmitNorms field bit should not be set.", !fi.fieldInfo("f1").omitNorms); assertTrue("OmitNorms field bit should not be set.", !fi.fieldInfo("f1").omitNorms);
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms); assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);

2
lucene/src/test/org/apache/lucene/index/TestOmitPositions.java
View File

@ -154,7 +154,7 @@ public class TestOmitPositions extends LuceneTestCase {
writer.close(); writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
// docs + docs = docs // docs + docs = docs
assertEquals(IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions); assertEquals(IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
// docs + docs/freqs = docs // docs + docs/freqs = docs

6
lucene/src/test/org/apache/lucene/index/TestOmitTf.java
View File

@ -101,7 +101,7 @@ public class TestOmitTf extends LuceneTestCase {
writer.close(); writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
@ -153,7 +153,7 @@ public class TestOmitTf extends LuceneTestCase {
writer.close(); writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
@ -196,7 +196,7 @@ public class TestOmitTf extends LuceneTestCase {
writer.close(); writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions);
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions); assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);

14
lucene/src/test/org/apache/lucene/index/TestParallelReader.java
View File

@ -18,14 +18,12 @@ package org.apache.lucene.index;
*/ */
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.Random; import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField; import org.apache.lucene.document.TextField;
import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.*; import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
@ -75,12 +73,12 @@ public class TestParallelReader extends LuceneTestCase {
ParallelReader pr = new ParallelReader(); ParallelReader pr = new ParallelReader();
pr.add(IndexReader.open(dir1)); pr.add(IndexReader.open(dir1));
pr.add(IndexReader.open(dir2)); pr.add(IndexReader.open(dir2));
Collection<String> fieldNames = pr.getFieldNames(IndexReader.FieldOption.ALL); FieldInfos fieldInfos = pr.getFieldInfos();
assertEquals(4, fieldNames.size()); assertEquals(4, fieldInfos.size());
assertTrue(fieldNames.contains("f1")); assertNotNull(fieldInfos.fieldInfo("f1"));
assertTrue(fieldNames.contains("f2")); assertNotNull(fieldInfos.fieldInfo("f2"));
assertTrue(fieldNames.contains("f3")); assertNotNull(fieldInfos.fieldInfo("f3"));
assertTrue(fieldNames.contains("f4")); assertNotNull(fieldInfos.fieldInfo("f4"));
pr.close(); pr.close();
dir1.close(); dir1.close();
dir2.close(); dir2.close();

4
lucene/src/test/org/apache/lucene/index/TestPayloads.java
View File

@ -112,7 +112,7 @@ public class TestPayloads extends LuceneTestCase {
writer.close(); writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram)); SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
FieldInfos fi = reader.fieldInfos(); FieldInfos fi = reader.getFieldInfos();
assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads);
assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads); assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads);
assertFalse("Payload field bit should not be set.", fi.fieldInfo("f3").storePayloads); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f3").storePayloads);
@ -139,7 +139,7 @@ public class TestPayloads extends LuceneTestCase {
writer.close(); writer.close();
reader = getOnlySegmentReader(IndexReader.open(ram)); reader = getOnlySegmentReader(IndexReader.open(ram));
fi = reader.fieldInfos(); fi = reader.getFieldInfos();
assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads); assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads);
assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads); assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads);
assertTrue("Payload field bit should be set.", fi.fieldInfo("f3").storePayloads); assertTrue("Payload field bit should be set.", fi.fieldInfo("f3").storePayloads);

13
lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/ */
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.Codec;
@ -30,6 +29,7 @@ import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil; import org.apache.lucene.util._TestUtil;
public class TestSegmentMerger extends LuceneTestCase { public class TestSegmentMerger extends LuceneTestCase {
//The variables for the new merged segment //The variables for the new merged segment
private Directory mergedDir; private Directory mergedDir;
@ -107,10 +107,15 @@ public class TestSegmentMerger extends LuceneTestCase {
assertTrue(termDocs != null); assertTrue(termDocs != null);
assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS); assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
Collection<String> stored = mergedReader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR); int tvCount = 0;
assertTrue(stored != null); for(FieldInfo fieldInfo : mergedReader.getFieldInfos()) {
if (fieldInfo.storeTermVector) {
tvCount++;
}
}
//System.out.println("stored size: " + stored.size()); //System.out.println("stored size: " + stored.size());
assertTrue("We do not have 3 fields that were indexed with term vector",stored.size() == 3); assertEquals("We do not have 3 fields that were indexed with term vector", 3, tvCount);
Terms vector = mergedReader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY); Terms vector = mergedReader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
assertNotNull(vector); assertNotNull(vector);

51
lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException; import java.io.IOException;
import java.util.Collection; import java.util.Collection;
import java.util.Iterator; import java.util.HashSet;
import java.util.List; import java.util.List;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
@ -74,33 +74,42 @@ public class TestSegmentReader extends LuceneTestCase {
} }
public void testGetFieldNameVariations() { public void testGetFieldNameVariations() {
Collection<String> result = reader.getFieldNames(IndexReader.FieldOption.ALL); Collection<String> allFieldNames = new HashSet<String>();
assertTrue(result != null); Collection<String> indexedFieldNames = new HashSet<String>();
assertTrue(result.size() == DocHelper.all.size()); Collection<String> notIndexedFieldNames = new HashSet<String>();
for (Iterator<String> iter = result.iterator(); iter.hasNext();) { Collection<String> tvFieldNames = new HashSet<String>();
String s = iter.next(); Collection<String> noTVFieldNames = new HashSet<String>();
//System.out.println("Name: " + s);
for(FieldInfo fieldInfo : reader.getFieldInfos()) {
final String name = fieldInfo.name;
allFieldNames.add(name);
if (fieldInfo.isIndexed) {
indexedFieldNames.add(name);
} else {
notIndexedFieldNames.add(name);
}
if (fieldInfo.storeTermVector) {
tvFieldNames.add(name);
} else if (fieldInfo.isIndexed) {
noTVFieldNames.add(name);
}
}
assertTrue(allFieldNames.size() == DocHelper.all.size());
for (String s : allFieldNames) {
assertTrue(DocHelper.nameValues.containsKey(s) == true || s.equals("")); assertTrue(DocHelper.nameValues.containsKey(s) == true || s.equals(""));
} }
result = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
assertTrue(result != null); assertTrue(indexedFieldNames.size() == DocHelper.indexed.size());
assertTrue(result.size() == DocHelper.indexed.size()); for (String s : indexedFieldNames) {
for (Iterator<String> iter = result.iterator(); iter.hasNext();) {
String s = iter.next();
assertTrue(DocHelper.indexed.containsKey(s) == true || s.equals("")); assertTrue(DocHelper.indexed.containsKey(s) == true || s.equals(""));
} }
result = reader.getFieldNames(IndexReader.FieldOption.UNINDEXED); assertTrue(notIndexedFieldNames.size() == DocHelper.unindexed.size());
assertTrue(result != null);
assertTrue(result.size() == DocHelper.unindexed.size());
//Get all indexed fields that are storing term vectors //Get all indexed fields that are storing term vectors
result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR); assertTrue(tvFieldNames.size() == DocHelper.termvector.size());
assertTrue(result != null);
assertTrue(result.size() == DocHelper.termvector.size());
result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_NO_TERMVECTOR); assertTrue(noTVFieldNames.size() == DocHelper.notermvector.size());
assertTrue(result != null);
assertTrue(result.size() == DocHelper.notermvector.size());
} }
public void testTerms() throws IOException { public void testTerms() throws IOException {

31
modules/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
View File

@ -16,22 +16,23 @@ package org.apache.lucene.analysis.query;
* limitations under the License. * limitations under the License.
*/ */
import org.apache.lucene.analysis.AnalyzerWrapper;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.BytesRef;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.AnalyzerWrapper;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.Version;
/** /**
* An {@link Analyzer} used primarily at query time to wrap another analyzer and provide a layer of protection * An {@link Analyzer} used primarily at query time to wrap another analyzer and provide a layer of protection
* which prevents very common words from being passed into queries. * which prevents very common words from being passed into queries.
@ -84,7 +85,7 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
Analyzer delegate, Analyzer delegate,
IndexReader indexReader, IndexReader indexReader,
int maxDocFreq) throws IOException { int maxDocFreq) throws IOException {
this(matchVersion, delegate, indexReader, indexReader.getFieldNames(IndexReader.FieldOption.INDEXED), maxDocFreq); this(matchVersion, delegate, indexReader, ReaderUtil.getIndexedFields(indexReader), maxDocFreq);
} }
/** /**
@ -104,7 +105,7 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
Analyzer delegate, Analyzer delegate,
IndexReader indexReader, IndexReader indexReader,
float maxPercentDocs) throws IOException { float maxPercentDocs) throws IOException {
this(matchVersion, delegate, indexReader, indexReader.getFieldNames(IndexReader.FieldOption.INDEXED), maxPercentDocs); this(matchVersion, delegate, indexReader, ReaderUtil.getIndexedFields(indexReader), maxPercentDocs);
} }
/** /**

3
modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
View File

@ -34,6 +34,7 @@ import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util.UnicodeUtil;
@ -569,7 +570,7 @@ public final class MoreLikeThis {
public Query like(int docNum) throws IOException { public Query like(int docNum) throws IOException {
if (fieldNames == null) { if (fieldNames == null) {
// gather list of valid fields from lucene // gather list of valid fields from lucene
Collection<String> fields = ir.getFieldNames(IndexReader.FieldOption.INDEXED); Collection<String> fields = ReaderUtil.getIndexedFields(ir);
fieldNames = fields.toArray(new String[fields.size()]); fieldNames = fields.toArray(new String[fields.size()]);
} }

14
solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
View File

@ -27,13 +27,11 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.index.*; import org.apache.lucene.index.*;
import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo.IndexOptions;
import static org.apache.lucene.index.FieldInfo.IndexOptions.DOCS_ONLY;
import static org.apache.lucene.index.FieldInfo.IndexOptions.DOCS_AND_FREQS;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.analysis.CharFilterFactory; import org.apache.solr.analysis.CharFilterFactory;
import org.apache.solr.analysis.TokenFilterFactory; import org.apache.solr.analysis.TokenFilterFactory;
@ -55,7 +53,9 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrIndexSearcher;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.lucene.index.DocsEnum;
import static org.apache.lucene.index.FieldInfo.IndexOptions.DOCS_AND_FREQS;
import static org.apache.lucene.index.FieldInfo.IndexOptions.DOCS_ONLY;
/** /**
* This handler exposes the internal lucene index. It is inspired by and * This handler exposes the internal lucene index. It is inspired by and
@ -289,11 +289,15 @@ public class LukeRequestHandler extends RequestHandlerBase
IndexReader reader = searcher.getIndexReader(); IndexReader reader = searcher.getIndexReader();
IndexSchema schema = searcher.getSchema(); IndexSchema schema = searcher.getSchema();
Set<String> fieldNames = new TreeSet<String>();
for(FieldInfo fieldInfo : ReaderUtil.getMergedFieldInfos(reader)) {
fieldNames.add(fieldInfo.name);
}
// Walk the term enum and keep a priority queue for each map in our set // Walk the term enum and keep a priority queue for each map in our set
SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>(); SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>();
Fields theFields = MultiFields.getFields(reader); Fields theFields = MultiFields.getFields(reader);
Set<String> fieldNames = new TreeSet<String>(reader.getFieldNames(IndexReader.FieldOption.ALL));
for (String fieldName : fieldNames) { for (String fieldName : fieldNames) {
if (fields != null && ! fields.contains(fieldName)) { if (fields != null && ! fields.contains(fieldName)) {
continue; // we're not interested in this term continue; // we're not interested in this term

8
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -38,6 +38,7 @@ import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Bits; import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.ReaderUtil;
import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
@ -180,7 +181,10 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
} }
optimizer = solrConfig.filtOptEnabled ? new LuceneQueryOptimizer(solrConfig.filtOptCacheSize,solrConfig.filtOptThreshold) : null; optimizer = solrConfig.filtOptEnabled ? new LuceneQueryOptimizer(solrConfig.filtOptCacheSize,solrConfig.filtOptThreshold) : null;
fieldNames = r.getFieldNames(IndexReader.FieldOption.ALL); fieldNames = new HashSet<String>();
for(FieldInfo fieldInfo : ReaderUtil.getMergedFieldInfos(r)) {
fieldNames.add(fieldInfo.name);
}
// do this at the end since an exception in the constructor means we won't close // do this at the end since an exception in the constructor means we won't close
numOpens.incrementAndGet(); numOpens.incrementAndGet();
@ -421,8 +425,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
public void stringField(FieldInfo fieldInfo, String value) throws IOException { public void stringField(FieldInfo fieldInfo, String value) throws IOException {
final FieldType ft = new FieldType(TextField.TYPE_STORED); final FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.storeTermVector); ft.setStoreTermVectors(fieldInfo.storeTermVector);
ft.setStoreTermVectorPositions(fieldInfo.storePositionWithTermVector);
ft.setStoreTermVectorOffsets(fieldInfo.storeOffsetWithTermVector);
ft.setStoreTermVectors(fieldInfo.storeTermVector); ft.setStoreTermVectors(fieldInfo.storeTermVector);
ft.setIndexed(fieldInfo.isIndexed); ft.setIndexed(fieldInfo.isIndexed);
ft.setOmitNorms(fieldInfo.omitNorms); ft.setOmitNorms(fieldInfo.omitNorms);

20
solr/core/src/test/org/apache/solr/search/TestDocSet.java
View File

@ -17,22 +17,23 @@
package org.apache.solr.search; package org.apache.solr.search;
import java.util.Random;
import java.util.Arrays;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.util.OpenBitSetIterator;
import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext; import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.MultiReader;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.OpenBitSetIterator;
/** /**
* *
@ -358,6 +359,11 @@ public class TestDocSet extends LuceneTestCase {
public IndexReader[] getSequentialSubReaders() { public IndexReader[] getSequentialSubReaders() {
return null; return null;
} }
@Override
public FieldInfos getFieldInfos() {
return new FieldInfos();
}
}; };
return r; return r;
} }