Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3622: simplify how indexing chain makes changes to FIS 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3622@1211601 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2011-12-07 19:06:49 +00:00
parent 36b8f12325
commit d6fd85f4c0
15 changed files with 54 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -1140,7 +1140,7 @@ public class CheckIndex {
final Source values = docValues.getDirectSource();
final int maxDoc = reader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
switch (fieldInfo.docValues) {
switch (fieldInfo.getDocValuesType()) {
case BYTES_FIXED_SORTED:
case BYTES_VAR_SORTED:
case BYTES_FIXED_DEREF:
@ -1162,7 +1162,7 @@ public class CheckIndex {
break;
default:
throw new IllegalArgumentException("Field: " + fieldInfo.name
+ " - no such DocValues type: " + fieldInfo.docValues);
+ " - no such DocValues type: " + fieldInfo.getDocValuesType());
}
}
}

25
lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
View File

@ -26,11 +26,12 @@ import java.util.Map;
import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.DocValuesFormat;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.DocValuesFormat;
import org.apache.lucene.index.codecs.FieldInfosWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.values.PerDocFieldValues;
import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IOUtils;
@ -224,7 +225,7 @@ final class DocFieldProcessor extends DocConsumer {
// needs to be more "pluggable" such that if I want
// to have a new "thing" my Fields can do, I can
// easily add it
FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.fieldType(), false, field.docValuesType());
FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.fieldType());
fp = new DocFieldProcessorPerField(this, fi);
fp.next = fieldHash[hashPos];
@ -235,7 +236,7 @@ final class DocFieldProcessor extends DocConsumer {
rehash();
}
} else {
fieldInfos.addOrUpdate(fp.fieldInfo.name, field.fieldType(), false, field.docValuesType());
fieldInfos.addOrUpdate(fp.fieldInfo.name, field.fieldType());
}
if (thisFieldGen != fp.lastGen) {
@ -261,7 +262,7 @@ final class DocFieldProcessor extends DocConsumer {
}
final PerDocFieldValues docValues = field.docValues();
if (docValues != null) {
docValuesConsumer(docState, fp.fieldInfo).add(docState.docID, docValues);
docValuesConsumer(field.docValuesType(), docState, fp.fieldInfo).add(docState.docID, docValues);
}
}
@ -310,7 +311,7 @@ final class DocFieldProcessor extends DocConsumer {
final private Map<String, DocValuesConsumerAndDocID> docValues = new HashMap<String, DocValuesConsumerAndDocID>();
final private Map<Integer, PerDocConsumer> perDocConsumers = new HashMap<Integer, PerDocConsumer>();
DocValuesConsumer docValuesConsumer(DocState docState, FieldInfo fieldInfo)
DocValuesConsumer docValuesConsumer(ValueType valueType, DocState docState, FieldInfo fieldInfo)
throws IOException {
DocValuesConsumerAndDocID docValuesConsumerAndDocID = docValues.get(fieldInfo.name);
if (docValuesConsumerAndDocID != null) {
@ -329,17 +330,9 @@ final class DocFieldProcessor extends DocConsumer {
perDocConsumer = dvFormat.docsConsumer(perDocWriteState);
perDocConsumers.put(0, perDocConsumer);
}
boolean success = false;
DocValuesConsumer docValuesConsumer = null;
try {
docValuesConsumer = perDocConsumer.addValuesField(fieldInfo);
fieldInfo.commitDocValues();
success = true;
} finally {
if (!success) {
fieldInfo.revertUncommitted();
}
}
DocValuesConsumer docValuesConsumer = perDocConsumer.addValuesField(valueType, fieldInfo);
fieldInfo.setDocValuesType(valueType);
docValuesConsumerAndDocID = new DocValuesConsumerAndDocID(docValuesConsumer);
docValuesConsumerAndDocID.docID = docState.docID;

2
lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
View File

@ -244,7 +244,6 @@ public class DocumentsWriterPerThread {
// mark document as deleted
deleteDocID(docState.docID);
numDocsInRAM++;
fieldInfos.revertUncommitted();
} else {
abort();
}
@ -308,7 +307,6 @@ public class DocumentsWriterPerThread {
// Incr here because finishDocument will not
// be called (because an exc is being thrown):
numDocsInRAM++;
fieldInfos.revertUncommitted();
} else {
abort();
}

46
lucene/src/java/org/apache/lucene/index/FieldInfo.java
View File

@ -25,7 +25,7 @@ public final class FieldInfo {
public final int number;
public boolean isIndexed;
ValueType docValues;
private ValueType docValues;
// true if term vector for this field should be stored
@ -118,13 +118,14 @@ public final class FieldInfo {
}
assert this.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !this.storePayloads;
}
void setDocValues(ValueType v) {
void setDocValuesType(ValueType v) {
if (docValues == null) {
docValues = v;
}
}
public void resetDocValues(ValueType v) {
public void resetDocValuesType(ValueType v) {
if (docValues != null) {
docValues = v;
}
@ -134,42 +135,13 @@ public final class FieldInfo {
return docValues != null;
}
public ValueType getDocValues() {
public ValueType getDocValuesType() {
return docValues;
}
private boolean vectorsCommitted;
private boolean docValuesCommitted;
/**
* Reverts all uncommitted changes on this {@link FieldInfo}
* @see #commitVectors()
*/
void revertUncommitted() {
if (storeTermVector && !vectorsCommitted) {
storeOffsetWithTermVector = false;
storePositionWithTermVector = false;
storeTermVector = false;
}
if (docValues != null && !docValuesCommitted) {
docValues = null;
}
}
/**
* Commits term vector modifications. Changes to term-vectors must be
* explicitly committed once the necessary files are created. If those changes
* are not committed subsequent {@link #revertUncommitted()} will reset the
* all term-vector flags before the next document.
*/
void commitVectors() {
assert storeTermVector;
vectorsCommitted = true;
}
void commitDocValues() {
assert hasDocValues();
docValuesCommitted = true;
public void setStoreTermVectors(boolean withPositions, boolean withOffsets) {
storeTermVector = true;
storePositionWithTermVector |= withPositions;
storeOffsetWithTermVector |= withOffsets;
}
}

32
lucene/src/java/org/apache/lucene/index/FieldInfos.java
View File

@ -304,10 +304,20 @@ public final class FieldInfos implements Iterable<FieldInfo> {
storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues);
}
synchronized public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType, boolean scorePayloads, ValueType docValues) {
return addOrUpdateInternal(name, -1, fieldType.indexed(), fieldType.storeTermVectors(),
fieldType.storeTermVectorPositions(), fieldType.storeTermVectorOffsets(), fieldType.omitNorms(), scorePayloads,
fieldType.indexOptions(), docValues);
// NOTE: this method does not carry over termVector
// booleans nor docValuesType; the indexer chain
// (TermVectorsConsumerPerField, DocFieldProcessor) must
// set these fields when they succeed in consuming
// the document:
public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
// TODO: really, indexer shouldn't even call this
// method (it's only called from DocFieldProcessor);
// rather, each component in the chain should update
// what it "owns". EG fieldType.indexOptions() should
// be updated by maybe FreqProxTermsWriterPerField:
return addOrUpdateInternal(name, -1, fieldType.indexed(), false, false, false,
fieldType.omitNorms(), false,
fieldType.indexOptions(), null);
}
synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
@ -322,7 +332,7 @@ public final class FieldInfos implements Iterable<FieldInfo> {
fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues);
} else {
fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions);
fi.setDocValues(docValues);
fi.setDocValuesType(docValues);
}
version++;
return fi;
@ -333,7 +343,7 @@ public final class FieldInfos implements Iterable<FieldInfo> {
return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed, fi.storeTermVector,
fi.storePositionWithTermVector, fi.storeOffsetWithTermVector,
fi.omitNorms, fi.storePayloads,
fi.indexOptions, fi.docValues);
fi.indexOptions, fi.getDocValuesType());
}
/*
@ -429,16 +439,6 @@ public final class FieldInfos implements Iterable<FieldInfo> {
return version;
}
/**
* Reverts all uncommitted changes
* @see FieldInfo#revertUncommitted()
*/
void revertUncommitted() {
for (FieldInfo fieldInfo : this) {
fieldInfo.revertUncommitted();
}
}
final FieldInfos asReadOnly() {
if (isReadOnly()) {
return this;

8
lucene/src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -235,7 +235,7 @@ final class SegmentMerger {
for (String dvName : dvNames) {
FieldInfo merged = mergeState.fieldInfos.fieldInfo(dvName);
IndexDocValues docValues = reader.docValues(dvName);
merged.setDocValues(docValues.type());
merged.setDocValuesType(docValues.type());
TypePromoter previous = docValuesTypes.get(merged);
docValuesTypes.put(merged, mergeDocValuesType(previous, docValues));
}
@ -247,12 +247,12 @@ final class SegmentMerger {
FieldInfo fi = e.getKey();
TypePromoter promoter = e.getValue();
if (promoter == null) {
fi.resetDocValues(null);
fi.resetDocValuesType(null);
} else {
assert promoter != TypePromoter.getIdentityPromoter();
if (fi.getDocValues() != promoter.type()) {
if (fi.getDocValuesType() != promoter.type()) {
// reset the type if we got promoted
fi.resetDocValues(promoter.type());
fi.resetDocValuesType(promoter.type());
}
}
}

4
lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
View File

@ -113,7 +113,6 @@ final class TermVectorsConsumerPerField extends TermsHashConsumerPerField {
// of a given field in the doc. At this point we flush
// our hash into the DocWriter.
assert fieldInfo.storeTermVector;
assert termsWriter.vectorFieldsInOrder(fieldInfo);
TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
@ -150,8 +149,9 @@ final class TermVectorsConsumerPerField extends TermsHashConsumerPerField {
}
termsHashPerField.reset();
// commit the termVectors once successful success - FI will otherwise reset them
fieldInfo.commitVectors();
fieldInfo.setStoreTermVectors(doVectorPositions, doVectorOffsets);
}
void shrinkHash() {

2
lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java
View File

@ -74,7 +74,7 @@ public abstract class DocValuesReaderBase extends PerDocValues {
final String id = DocValuesWriterBase.docValuesId(segment,
fieldInfo.number);
values.put(field,
loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context));
loadDocValues(docCount, dir, id, fieldInfo.getDocValuesType(), context));
}
}
success = true;

5
lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java
View File

@ -23,6 +23,7 @@ import java.util.Comparator;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.codecs.lucene40.values.Writer;
import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
@ -52,8 +53,8 @@ public abstract class DocValuesWriterBase extends PerDocConsumer {
}
@Override
public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
return Writer.create(field.getDocValues(),
public DocValuesConsumer addValuesField(ValueType valueType, FieldInfo field) throws IOException {
return Writer.create(valueType,
docValuesId(segmentName, field.number),
getDirectory(), getComparator(), bytesUsed, context);
}

5
lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.index.values.ValueType;
/**
* Abstract API that consumes per document values. Concrete implementations of
@ -34,7 +35,7 @@ import org.apache.lucene.index.values.IndexDocValues;
*/
public abstract class PerDocConsumer implements Closeable{
/** Adds a new DocValuesField */
public abstract DocValuesConsumer addValuesField(FieldInfo field)
public abstract DocValuesConsumer addValuesField(ValueType type, FieldInfo field)
throws IOException;
/**
@ -56,7 +57,7 @@ public abstract class PerDocConsumer implements Closeable{
docValues[i] = perDocValues[i].docValues(fieldInfo.name);
}
}
final DocValuesConsumer docValuesConsumer = addValuesField(fieldInfo);
final DocValuesConsumer docValuesConsumer = addValuesField(fieldInfo.getDocValuesType(), fieldInfo);
assert docValuesConsumer != null;
docValuesConsumer.merge(mergeState, docValues);
}

4
lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40FieldInfosWriter.java
View File

@ -83,7 +83,7 @@ public class Lucene40FieldInfosWriter extends FieldInfosWriter {
if (!fi.hasDocValues()) {
b = 0;
} else {
switch(fi.getDocValues()) {
switch(fi.getDocValuesType()) {
case VAR_INTS:
b = 1;
break;
@ -124,7 +124,7 @@ public class Lucene40FieldInfosWriter extends FieldInfosWriter {
b = 13;
break;
default:
throw new IllegalStateException("unhandled indexValues type " + fi.getDocValues());
throw new IllegalStateException("unhandled indexValues type " + fi.getDocValuesType());
}
}
output.writeByte(b);

2
lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java
View File

@ -53,7 +53,7 @@ public class SepDocValuesConsumer extends DocValuesWriterBase {
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.hasDocValues()) {
String filename = docValuesId(segmentInfo.name, fieldInfo.number);
switch (fieldInfo.getDocValues()) {
switch (fieldInfo.getDocValuesType()) {
case BYTES_FIXED_DEREF:
case BYTES_VAR_DEREF:
case BYTES_VAR_STRAIGHT:

2
lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldInfosWriter.java
View File

@ -100,7 +100,7 @@ public class SimpleTextFieldInfosWriter extends FieldInfosWriter {
if (!fi.hasDocValues()) {
SimpleTextUtil.write(out, "false", scratch);
} else {
SimpleTextUtil.write(out, fi.getDocValues().toString(), scratch);
SimpleTextUtil.write(out, fi.getDocValuesType().toString(), scratch);
}
SimpleTextUtil.writeNewline(out);

2
lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java
View File

@ -440,7 +440,7 @@ public class _TestUtil {
/** Adds field info for a Document. */
public static void add(Document doc, FieldInfos fieldInfos) {
for (IndexableField field : doc) {
fieldInfos.addOrUpdate(field.name(), field.fieldType(), false, field.docValuesType());
fieldInfos.addOrUpdate(field.name(), field.fieldType());
}
}

2
lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
View File

@ -27,7 +27,6 @@ import org.apache.lucene.index.codecs.FieldInfosWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.junit.Ignore;
import java.io.IOException;
import java.util.Arrays;
@ -82,7 +81,6 @@ public class TestFieldInfos extends LuceneTestCase {
info = readIn.fieldInfo("textField2");
assertTrue(info != null);
assertTrue(info.storeTermVector == true);
assertTrue(info.omitNorms == false);
info = readIn.fieldInfo("textField3");