Change docValuesSkipIndex from a boolean to an enum. (#13784)

At the moment, our skip indexes record min/max ordinal/value per range
of doc IDs. It would be natural to extend it to other pre-aggregated
data such as a sum and value count, which facets could take advantage
of. This change switches `docValuesSkipIndex` from a boolean to an enum
so that we could release such changes in the future in an additive
fashion, by adding constants to this enum and new methods to
`DocValuesSkipper`.
This commit is contained in:
Adrien Grand 2024-09-17 14:35:30 +02:00 committed by GitHub
parent 644feeb02a
commit b59a357e58
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
39 changed files with 214 additions and 108 deletions

View File

@ -24,6 +24,7 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -209,7 +210,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
storePayloads, storePayloads,
indexOptions, indexOptions,
docValuesType, docValuesType,
false, DocValuesSkipIndexType.NONE,
dvGen, dvGen,
attributes, attributes,
pointDataDimensionCount, pointDataDimensionCount,

View File

@ -23,6 +23,7 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -186,7 +187,7 @@ public final class Lucene90FieldInfosFormat extends FieldInfosFormat {
storePayloads, storePayloads,
indexOptions, indexOptions,
docValuesType, docValuesType,
false, DocValuesSkipIndexType.NONE,
dvGen, dvGen,
attributes, attributes,
pointDataDimensionCount, pointDataDimensionCount,

View File

@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -125,8 +126,8 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
SimpleTextUtil.readLine(input, scratch); SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), DOCVALUES_SKIP_INDEX); assert StringHelper.startsWith(scratch.get(), DOCVALUES_SKIP_INDEX);
boolean docValueSkipper = DocValuesSkipIndexType docValueSkipper =
Boolean.parseBoolean(readString(DOCVALUES_SKIP_INDEX.length, scratch)); docValuesSkipIndexType(readString(DOCVALUES_SKIP_INDEX.length, scratch));
SimpleTextUtil.readLine(input, scratch); SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), DOCVALUES_GEN); assert StringHelper.startsWith(scratch.get(), DOCVALUES_GEN);
@ -221,6 +222,10 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
return DocValuesType.valueOf(dvType); return DocValuesType.valueOf(dvType);
} }
public DocValuesSkipIndexType docValuesSkipIndexType(String dvSkipIndexType) {
return DocValuesSkipIndexType.valueOf(dvSkipIndexType);
}
public VectorEncoding vectorEncoding(String vectorEncoding) { public VectorEncoding vectorEncoding(String vectorEncoding) {
return VectorEncoding.valueOf(vectorEncoding); return VectorEncoding.valueOf(vectorEncoding);
} }
@ -284,7 +289,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
SimpleTextUtil.writeNewline(out); SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, DOCVALUES_SKIP_INDEX); SimpleTextUtil.write(out, DOCVALUES_SKIP_INDEX);
SimpleTextUtil.write(out, Boolean.toString(fi.hasDocValuesSkipIndex()), scratch); SimpleTextUtil.write(out, getDocValuesSkipIndexType(fi.docValuesSkipIndexType()), scratch);
SimpleTextUtil.writeNewline(out); SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, DOCVALUES_GEN); SimpleTextUtil.write(out, DOCVALUES_GEN);
@ -355,4 +360,8 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
private static String getDocValuesType(DocValuesType type) { private static String getDocValuesType(DocValuesType type) {
return type.toString(); return type.toString();
} }
private static String getDocValuesSkipIndexType(DocValuesSkipIndexType type) {
return type.toString();
}
} }

View File

@ -20,6 +20,7 @@ package org.apache.lucene.codecs.uniformsplit;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import org.apache.lucene.codecs.lucene90.tests.MockTermStateFactory; import org.apache.lucene.codecs.lucene90.tests.MockTermStateFactory;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexOptions;
@ -111,7 +112,7 @@ public class TestBlockWriter extends LuceneTestCase {
true, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
Collections.emptyMap(), Collections.emptyMap(),
0, 0,

View File

@ -34,6 +34,7 @@ import org.apache.lucene.codecs.uniformsplit.FSTDictionary;
import org.apache.lucene.codecs.uniformsplit.FieldMetadata; import org.apache.lucene.codecs.uniformsplit.FieldMetadata;
import org.apache.lucene.codecs.uniformsplit.IndexDictionary; import org.apache.lucene.codecs.uniformsplit.IndexDictionary;
import org.apache.lucene.codecs.uniformsplit.TermBytes; import org.apache.lucene.codecs.uniformsplit.TermBytes;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -198,7 +199,7 @@ public class TestSTBlockReader extends LuceneTestCase {
true, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
Collections.emptyMap(), Collections.emptyMap(),
0, 0,

View File

@ -19,6 +19,7 @@ package org.apache.lucene.codecs;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
@ -77,7 +78,7 @@ public abstract class DocValuesProducer implements Closeable {
/** /**
* Returns a {@link DocValuesSkipper} for this field. The returned instance need not be * Returns a {@link DocValuesSkipper} for this field. The returned instance need not be
* thread-safe: it will only be used by a single thread. The return value is undefined if {@link * thread-safe: it will only be used by a single thread. The return value is undefined if {@link
* FieldInfo#hasDocValuesSkipIndex()} doesn't return {@code true}. * FieldInfo#docValuesSkipIndexType()} returns {@link DocValuesSkipIndexType#NONE}.
*/ */
public abstract DocValuesSkipper getSkipper(FieldInfo field) throws IOException; public abstract DocValuesSkipper getSkipper(FieldInfo field) throws IOException;

View File

@ -31,6 +31,7 @@ import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.EmptyDocValuesProducer;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFileNames;
@ -143,7 +144,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
return DocValues.singleton(valuesProducer.getNumeric(field)); return DocValues.singleton(valuesProducer.getNumeric(field));
} }
}; };
if (field.hasDocValuesSkipIndex()) { if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
writeSkipIndex(field, producer); writeSkipIndex(field, producer);
} }
writeValues(field, producer, false); writeValues(field, producer, false);
@ -248,7 +249,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer) private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer)
throws IOException { throws IOException {
assert field.hasDocValuesSkipIndex(); assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE;
final long start = data.getFilePointer(); final long start = data.getFilePointer();
final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
long globalMaxValue = Long.MIN_VALUE; long globalMaxValue = Long.MIN_VALUE;
@ -700,7 +701,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
return DocValues.singleton(sortedOrds); return DocValues.singleton(sortedOrds);
} }
}; };
if (field.hasDocValuesSkipIndex()) { if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
writeSkipIndex(field, producer); writeSkipIndex(field, producer);
} }
if (addTypeByte) { if (addTypeByte) {
@ -873,7 +874,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
private void doAddSortedNumericField( private void doAddSortedNumericField(
FieldInfo field, DocValuesProducer valuesProducer, boolean ords) throws IOException { FieldInfo field, DocValuesProducer valuesProducer, boolean ords) throws IOException {
if (field.hasDocValuesSkipIndex()) { if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
writeSkipIndex(field, valuesProducer); writeSkipIndex(field, valuesProducer);
} }
if (ords) { if (ords) {

View File

@ -27,6 +27,7 @@ import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -191,7 +192,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
} }
byte type = meta.readByte(); byte type = meta.readByte();
if (info.hasDocValuesSkipIndex()) { if (info.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
skippers.put(info.number, readDocValueSkipperMeta(meta)); skippers.put(info.number, readDocValueSkipperMeta(meta));
} }
if (type == Lucene90DocValuesFormat.NUMERIC) { if (type == Lucene90DocValuesFormat.NUMERIC) {

View File

@ -24,6 +24,7 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -163,8 +164,6 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0; boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0;
boolean isParentField = boolean isParentField =
format >= FORMAT_PARENT_FIELD ? (bits & PARENT_FIELD_FIELD) != 0 : false; format >= FORMAT_PARENT_FIELD ? (bits & PARENT_FIELD_FIELD) != 0 : false;
boolean hasDocValuesSkipIndex =
format >= FORMAT_DOCVALUE_SKIPPER ? (bits & DOCVALUES_SKIPPER) != 0 : false;
if ((bits & 0xC0) != 0) { if ((bits & 0xC0) != 0) {
throw new CorruptIndexException( throw new CorruptIndexException(
@ -187,6 +186,12 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
// DV Types are packed in one byte // DV Types are packed in one byte
final DocValuesType docValuesType = getDocValuesType(input, input.readByte()); final DocValuesType docValuesType = getDocValuesType(input, input.readByte());
final DocValuesSkipIndexType docValuesSkipIndex;
if (format >= FORMAT_DOCVALUE_SKIPPER) {
docValuesSkipIndex = getDocValuesSkipIndexType(input, input.readByte());
} else {
docValuesSkipIndex = DocValuesSkipIndexType.NONE;
}
final long dvGen = input.readLong(); final long dvGen = input.readLong();
Map<String, String> attributes = input.readMapOfStrings(); Map<String, String> attributes = input.readMapOfStrings();
// just use the last field's map if its the same // just use the last field's map if its the same
@ -217,7 +222,7 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
storePayloads, storePayloads,
indexOptions, indexOptions,
docValuesType, docValuesType,
hasDocValuesSkipIndex, docValuesSkipIndex,
dvGen, dvGen,
attributes, attributes,
pointDataDimensionCount, pointDataDimensionCount,
@ -270,6 +275,18 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
} }
} }
private static byte docValuesSkipIndexByte(DocValuesSkipIndexType type) {
switch (type) {
case NONE:
return 0;
case RANGE:
return 1;
default:
// BUG
throw new AssertionError("unhandled DocValuesSkipIndexType: " + type);
}
}
private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException { private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
switch (b) { switch (b) {
case 0: case 0:
@ -289,6 +306,18 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
} }
} }
private static DocValuesSkipIndexType getDocValuesSkipIndexType(IndexInput input, byte b)
throws IOException {
switch (b) {
case 0:
return DocValuesSkipIndexType.NONE;
case 1:
return DocValuesSkipIndexType.RANGE;
default:
throw new CorruptIndexException("invalid docvaluesskipindex byte: " + b, input);
}
}
private static VectorEncoding getVectorEncoding(IndexInput input, byte b) throws IOException { private static VectorEncoding getVectorEncoding(IndexInput input, byte b) throws IOException {
if (b < 0 || b >= VectorEncoding.values().length) { if (b < 0 || b >= VectorEncoding.values().length) {
throw new CorruptIndexException("invalid vector encoding: " + b, input); throw new CorruptIndexException("invalid vector encoding: " + b, input);
@ -404,13 +433,13 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
if (fi.hasPayloads()) bits |= STORE_PAYLOADS; if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD; if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;
if (fi.isParentField()) bits |= PARENT_FIELD_FIELD; if (fi.isParentField()) bits |= PARENT_FIELD_FIELD;
if (fi.hasDocValuesSkipIndex()) bits |= DOCVALUES_SKIPPER;
output.writeByte(bits); output.writeByte(bits);
output.writeByte(indexOptionsByte(fi.getIndexOptions())); output.writeByte(indexOptionsByte(fi.getIndexOptions()));
// pack the DV type and hasNorms in one byte // pack the DV type and hasNorms in one byte
output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeByte(docValuesByte(fi.getDocValuesType()));
output.writeByte(docValuesSkipIndexByte(fi.docValuesSkipIndexType()));
output.writeLong(fi.getDocValuesGen()); output.writeLong(fi.getDocValuesGen());
output.writeMapOfStrings(fi.attributes()); output.writeMapOfStrings(fi.attributes());
output.writeVInt(fi.getPointDimensionCount()); output.writeVInt(fi.getPointDimensionCount());

View File

@ -20,6 +20,7 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import org.apache.lucene.analysis.Analyzer; // javadocs import org.apache.lucene.analysis.Analyzer; // javadocs
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig;
@ -41,7 +42,7 @@ public class FieldType implements IndexableFieldType {
private IndexOptions indexOptions = IndexOptions.NONE; private IndexOptions indexOptions = IndexOptions.NONE;
private boolean frozen; private boolean frozen;
private DocValuesType docValuesType = DocValuesType.NONE; private DocValuesType docValuesType = DocValuesType.NONE;
private boolean docValuesSkipIndex; private DocValuesSkipIndexType docValuesSkipIndex = DocValuesSkipIndexType.NONE;
private int dimensionCount; private int dimensionCount;
private int indexDimensionCount; private int indexDimensionCount;
private int dimensionNumBytes; private int dimensionNumBytes;
@ -61,7 +62,7 @@ public class FieldType implements IndexableFieldType {
this.omitNorms = ref.omitNorms(); this.omitNorms = ref.omitNorms();
this.indexOptions = ref.indexOptions(); this.indexOptions = ref.indexOptions();
this.docValuesType = ref.docValuesType(); this.docValuesType = ref.docValuesType();
this.docValuesSkipIndex = ref.hasDocValuesSkipIndex(); this.docValuesSkipIndex = ref.docValuesSkipIndexType();
this.dimensionCount = ref.pointDimensionCount(); this.dimensionCount = ref.pointDimensionCount();
this.indexDimensionCount = ref.pointIndexDimensionCount(); this.indexDimensionCount = ref.pointIndexDimensionCount();
this.dimensionNumBytes = ref.pointNumBytes(); this.dimensionNumBytes = ref.pointNumBytes();
@ -508,7 +509,7 @@ public class FieldType implements IndexableFieldType {
} }
@Override @Override
public boolean hasDocValuesSkipIndex() { public DocValuesSkipIndexType docValuesSkipIndexType() {
return docValuesSkipIndex; return docValuesSkipIndex;
} }
@ -518,7 +519,7 @@ public class FieldType implements IndexableFieldType {
* correlate with fields that are part of the index sort, so that values can be expected to be * correlate with fields that are part of the index sort, so that values can be expected to be
* clustered in the doc ID space. * clustered in the doc ID space.
*/ */
public void setDocValuesSkipIndex(boolean docValuesSkipIndex) { public void setDocValuesSkipIndexType(DocValuesSkipIndexType docValuesSkipIndex) {
checkIfFrozen(); checkIfFrozen();
this.docValuesSkipIndex = docValuesSkipIndex; this.docValuesSkipIndex = docValuesSkipIndex;
} }
@ -531,7 +532,7 @@ public class FieldType implements IndexableFieldType {
result = prime * result + indexDimensionCount; result = prime * result + indexDimensionCount;
result = prime * result + dimensionNumBytes; result = prime * result + dimensionNumBytes;
result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode()); result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode());
result = prime * result + Boolean.hashCode(docValuesSkipIndex); result = prime * result + (docValuesSkipIndex == null ? 0 : docValuesSkipIndex.hashCode());
result = prime * result + indexOptions.hashCode(); result = prime * result + indexOptions.hashCode();
result = prime * result + (omitNorms ? 1231 : 1237); result = prime * result + (omitNorms ? 1231 : 1237);
result = prime * result + (storeTermVectorOffsets ? 1231 : 1237); result = prime * result + (storeTermVectorOffsets ? 1231 : 1237);

View File

@ -16,6 +16,7 @@
*/ */
package org.apache.lucene.document; package org.apache.lucene.document;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
@ -42,13 +43,13 @@ public class NumericDocValuesField extends Field {
TYPE.freeze(); TYPE.freeze();
INDEXED_TYPE = new FieldType(TYPE); INDEXED_TYPE = new FieldType(TYPE);
INDEXED_TYPE.setDocValuesSkipIndex(true); INDEXED_TYPE.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE);
INDEXED_TYPE.freeze(); INDEXED_TYPE.freeze();
} }
/** /**
* Creates a new {@link NumericDocValuesField} with the specified 64-bit long value that also * Creates a new {@link NumericDocValuesField} with the specified 64-bit long value that also
* creates a {@link FieldType#hasDocValuesSkipIndex() skip index}. * creates a {@link FieldType#docValuesSkipIndexType() skip index}.
* *
* @param name field name * @param name field name
* @param value 64-bit long value * @param value 64-bit long value

View File

@ -17,6 +17,7 @@
package org.apache.lucene.document; package org.apache.lucene.document;
import java.util.Collection; import java.util.Collection;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.MultiTermQuery;
@ -48,13 +49,13 @@ public class SortedDocValuesField extends Field {
TYPE.freeze(); TYPE.freeze();
INDEXED_TYPE = new FieldType(TYPE); INDEXED_TYPE = new FieldType(TYPE);
INDEXED_TYPE.setDocValuesSkipIndex(true); INDEXED_TYPE.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE);
INDEXED_TYPE.freeze(); INDEXED_TYPE.freeze();
} }
/** /**
* Creates a new {@link SortedDocValuesField} with the specified 64-bit long value that also * Creates a new {@link SortedDocValuesField} with the specified 64-bit long value that also
* creates a {@link FieldType#hasDocValuesSkipIndex() skip index}. * creates a {@link FieldType#docValuesSkipIndexType() skip index}.
* *
* @param name field name * @param name field name
* @param bytes binary content * @param bytes binary content

View File

@ -16,6 +16,7 @@
*/ */
package org.apache.lucene.document; package org.apache.lucene.document;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
@ -50,13 +51,13 @@ public class SortedNumericDocValuesField extends Field {
TYPE.freeze(); TYPE.freeze();
INDEXED_TYPE = new FieldType(TYPE); INDEXED_TYPE = new FieldType(TYPE);
INDEXED_TYPE.setDocValuesSkipIndex(true); INDEXED_TYPE.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE);
INDEXED_TYPE.freeze(); INDEXED_TYPE.freeze();
} }
/** /**
* Creates a new {@link SortedNumericDocValuesField} with the specified 64-bit long value that * Creates a new {@link SortedNumericDocValuesField} with the specified 64-bit long value that
* also creates a {@link FieldType#hasDocValuesSkipIndex() skip index}. * also creates a {@link FieldType#docValuesSkipIndexType() skip index}.
* *
* @param name field name * @param name field name
* @param value 64-bit long value * @param value 64-bit long value

View File

@ -17,6 +17,7 @@
package org.apache.lucene.document; package org.apache.lucene.document;
import java.util.Collection; import java.util.Collection;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.MultiTermQuery;
@ -49,13 +50,13 @@ public class SortedSetDocValuesField extends Field {
TYPE.freeze(); TYPE.freeze();
INDEXED_TYPE = new FieldType(TYPE); INDEXED_TYPE = new FieldType(TYPE);
INDEXED_TYPE.setDocValuesSkipIndex(true); INDEXED_TYPE.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE);
INDEXED_TYPE.freeze(); INDEXED_TYPE.freeze();
} }
/** /**
* Creates a new {@link SortedSetDocValuesField} with the specified 64-bit long value that also * Creates a new {@link SortedSetDocValuesField} with the specified 64-bit long value that also
* creates a {@link FieldType#hasDocValuesSkipIndex() skip index}. * creates a {@link FieldType#docValuesSkipIndexType() skip index}.
* *
* @param name field name * @param name field name
* @param bytes binary content * @param bytes binary content

View File

@ -3731,7 +3731,7 @@ public final class CheckIndex implements Closeable {
private static void checkDocValues( private static void checkDocValues(
FieldInfo fi, DocValuesProducer dvReader, DocValuesStatus status) throws Exception { FieldInfo fi, DocValuesProducer dvReader, DocValuesStatus status) throws Exception {
if (fi.hasDocValuesSkipIndex()) { if (fi.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
status.totalSkippingIndex++; status.totalSkippingIndex++;
checkDocValueSkipper(fi, dvReader.getSkipper(fi)); checkDocValueSkipper(fi, dvReader.getSkipper(fi));
} }

View File

@ -200,7 +200,7 @@ public abstract class CodecReader extends LeafReader {
public final DocValuesSkipper getDocValuesSkipper(String field) throws IOException { public final DocValuesSkipper getDocValuesSkipper(String field) throws IOException {
ensureOpen(); ensureOpen();
FieldInfo fi = getFieldInfos().fieldInfo(field); FieldInfo fi = getFieldInfos().fieldInfo(field);
if (fi == null || fi.hasDocValuesSkipIndex() == false) { if (fi == null || fi.docValuesSkipIndexType() == DocValuesSkipIndexType.NONE) {
return null; return null;
} }
return getDocValuesReader().getSkipper(fi); return getDocValuesReader().getSkipper(fi);

View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
/** Options for skip indexes on doc values. */
public enum DocValuesSkipIndexType {
/** No skip index should be created. */
NONE {
@Override
boolean isCompatibleWith(DocValuesType dvType) {
return true;
}
},
/**
* Record range of values. This is suitable for {@link DocValuesType#NUMERIC}, {@link
* DocValuesType#SORTED_NUMERIC}, {@link DocValuesType#SORTED} and {@link
* DocValuesType#SORTED_SET} doc values, and will record the min/max values per range of doc IDs.
*/
RANGE {
@Override
boolean isCompatibleWith(DocValuesType dvType) {
return dvType == DocValuesType.NUMERIC
|| dvType == DocValuesType.SORTED_NUMERIC
|| dvType == DocValuesType.SORTED
|| dvType == DocValuesType.SORTED_SET;
}
};
// TODO: add support for pre-aggregated integer/float/double
abstract boolean isCompatibleWith(DocValuesType dvType);
}

View File

@ -22,37 +22,31 @@ package org.apache.lucene.index;
*/ */
public enum DocValuesType { public enum DocValuesType {
/** No doc values for this field. */ /** No doc values for this field. */
NONE(false), NONE,
/** A per-document Number */ /** A per-document Number */
NUMERIC(true), NUMERIC,
/** /**
* A per-document byte[]. Values may be larger than 32766 bytes, but different codecs may enforce * A per-document byte[]. Values may be larger than 32766 bytes, but different codecs may enforce
* their own limits. * their own limits.
*/ */
BINARY(false), BINARY,
/** /**
* A pre-sorted byte[]. Fields with this type only store distinct byte values and store an * A pre-sorted byte[]. Fields with this type only store distinct byte values and store an
* additional offset pointer per document to dereference the shared byte[]. The stored byte[] is * additional offset pointer per document to dereference the shared byte[]. The stored byte[] is
* presorted and allows access via document id, ordinal and by-value. Values must be {@code <= * presorted and allows access via document id, ordinal and by-value. Values must be {@code <=
* 32766} bytes. * 32766} bytes.
*/ */
SORTED(true), SORTED,
/** /**
* A pre-sorted Number[]. Fields with this type store numeric values in sorted order according to * A pre-sorted Number[]. Fields with this type store numeric values in sorted order according to
* {@link Long#compare(long, long)}. * {@link Long#compare(long, long)}.
*/ */
SORTED_NUMERIC(true), SORTED_NUMERIC,
/** /**
* A pre-sorted Set&lt;byte[]&gt;. Fields with this type only store distinct byte values and store * A pre-sorted Set&lt;byte[]&gt;. Fields with this type only store distinct byte values and store
* additional offset pointers per document to dereference the shared byte[]s. The stored byte[] is * additional offset pointers per document to dereference the shared byte[]s. The stored byte[] is
* presorted and allows access via document id, ordinal and by-value. Values must be {@code <= * presorted and allows access via document id, ordinal and by-value. Values must be {@code <=
* 32766} bytes. * 32766} bytes.
*/ */
SORTED_SET(true); SORTED_SET;
final boolean supportsSkipIndex; // pkg-private for use in FieldInfo
DocValuesType(boolean supportsSkipIndex) {
this.supportsSkipIndex = supportsSkipIndex;
}
} }

View File

@ -37,7 +37,7 @@ public final class FieldInfo {
private DocValuesType docValuesType = DocValuesType.NONE; private DocValuesType docValuesType = DocValuesType.NONE;
private final boolean docValuesSkipIndex; private final DocValuesSkipIndexType docValuesSkipIndex;
// True if any document indexed term vectors // True if any document indexed term vectors
private boolean storeTermVector; private boolean storeTermVector;
@ -83,7 +83,7 @@ public final class FieldInfo {
boolean storePayloads, boolean storePayloads,
IndexOptions indexOptions, IndexOptions indexOptions,
DocValuesType docValues, DocValuesType docValues,
boolean hasDocValuesSkipIndex, DocValuesSkipIndexType docValuesSkipIndex,
long dvGen, long dvGen,
Map<String, String> attributes, Map<String, String> attributes,
int pointDimensionCount, int pointDimensionCount,
@ -99,7 +99,7 @@ public final class FieldInfo {
this.docValuesType = this.docValuesType =
Objects.requireNonNull( Objects.requireNonNull(
docValues, "DocValuesType must not be null (field: \"" + name + "\")"); docValues, "DocValuesType must not be null (field: \"" + name + "\")");
this.docValuesSkipIndex = hasDocValuesSkipIndex; this.docValuesSkipIndex = docValuesSkipIndex;
this.indexOptions = this.indexOptions =
Objects.requireNonNull( Objects.requireNonNull(
indexOptions, "IndexOptions must not be null (field: \"" + name + "\")"); indexOptions, "IndexOptions must not be null (field: \"" + name + "\")");
@ -157,11 +157,13 @@ public final class FieldInfo {
if (docValuesType == null) { if (docValuesType == null) {
throw new IllegalArgumentException("DocValuesType must not be null (field: '" + name + "')"); throw new IllegalArgumentException("DocValuesType must not be null (field: '" + name + "')");
} }
if (docValuesType.supportsSkipIndex == false && docValuesSkipIndex) { if (docValuesSkipIndex.isCompatibleWith(docValuesType) == false) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"field '" "field '"
+ name + name
+ "' cannot have docValuesSkipIndex set to true with doc values type " + "' cannot have docValuesSkipIndexType="
+ docValuesSkipIndex
+ " with doc values type "
+ docValuesType); + docValuesType);
} }
if (dvGen != -1 && docValuesType == DocValuesType.NONE) { if (dvGen != -1 && docValuesType == DocValuesType.NONE) {
@ -308,14 +310,16 @@ public final class FieldInfo {
* @throws IllegalArgumentException if they are not the same * @throws IllegalArgumentException if they are not the same
*/ */
static void verifySameDocValuesSkipIndex( static void verifySameDocValuesSkipIndex(
String fieldName, boolean hasDocValuesSkipIndex1, boolean hasDocValuesSkipIndex2) { String fieldName,
DocValuesSkipIndexType hasDocValuesSkipIndex1,
DocValuesSkipIndexType hasDocValuesSkipIndex2) {
if (hasDocValuesSkipIndex1 != hasDocValuesSkipIndex2) { if (hasDocValuesSkipIndex1 != hasDocValuesSkipIndex2) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"cannot change field \"" "cannot change field \""
+ fieldName + fieldName
+ "\" from docValuesSkipIndex=" + "\" from docValuesSkipIndexType="
+ hasDocValuesSkipIndex1 + hasDocValuesSkipIndex1
+ " to inconsistent docValuesSkipIndex=" + " to inconsistent docValuesSkipIndexType="
+ hasDocValuesSkipIndex2); + hasDocValuesSkipIndex2);
} }
} }
@ -589,7 +593,7 @@ public final class FieldInfo {
} }
/** Returns true if, and only if, this field has a skip index. */ /** Returns true if, and only if, this field has a skip index. */
public boolean hasDocValuesSkipIndex() { public DocValuesSkipIndexType docValuesSkipIndexType() {
return docValuesSkipIndex; return docValuesSkipIndex;
} }

View File

@ -365,7 +365,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
IndexOptions indexOptions, IndexOptions indexOptions,
IndexOptionsProperties indexOptionsProperties, IndexOptionsProperties indexOptionsProperties,
DocValuesType docValuesType, DocValuesType docValuesType,
boolean docValuesSkipIndex, DocValuesSkipIndexType docValuesSkipIndex,
FieldDimensions fieldDimensions, FieldDimensions fieldDimensions,
FieldVectorProperties fieldVectorProperties) {} FieldVectorProperties fieldVectorProperties) {}
@ -444,7 +444,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
? new IndexOptionsProperties(fi.hasTermVectors(), fi.omitsNorms()) ? new IndexOptionsProperties(fi.hasTermVectors(), fi.omitsNorms())
: null, : null,
fi.getDocValuesType(), fi.getDocValuesType(),
fi.hasDocValuesSkipIndex(), fi.docValuesSkipIndexType(),
new FieldDimensions( new FieldDimensions(
fi.getPointDimensionCount(), fi.getPointDimensionCount(),
fi.getPointIndexDimensionCount(), fi.getPointIndexDimensionCount(),
@ -524,9 +524,9 @@ public class FieldInfos implements Iterable<FieldInfo> {
DocValuesType currentDVType = fieldProperties.docValuesType; DocValuesType currentDVType = fieldProperties.docValuesType;
verifySameDocValuesType(fieldName, currentDVType, fi.getDocValuesType()); verifySameDocValuesType(fieldName, currentDVType, fi.getDocValuesType());
boolean currentDocValuesSkipIndex = fieldProperties.docValuesSkipIndex; DocValuesSkipIndexType currentDocValuesSkipIndex = fieldProperties.docValuesSkipIndex;
verifySameDocValuesSkipIndex( verifySameDocValuesSkipIndex(
fieldName, currentDocValuesSkipIndex, fi.hasDocValuesSkipIndex()); fieldName, currentDocValuesSkipIndex, fi.docValuesSkipIndexType());
FieldDimensions dims = fieldProperties.fieldDimensions; FieldDimensions dims = fieldProperties.fieldDimensions;
verifySamePointsOptions( verifySamePointsOptions(
@ -582,7 +582,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
dvType, dvType,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,
@ -609,8 +609,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
+ fieldDvType + fieldDvType
+ "]."); + "].");
} }
boolean hasDocValuesSkipIndex = fieldProperties.docValuesSkipIndex; DocValuesSkipIndexType hasDocValuesSkipIndex = fieldProperties.docValuesSkipIndex;
if (hasDocValuesSkipIndex) { if (hasDocValuesSkipIndex != DocValuesSkipIndexType.NONE) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"Can't update [" "Can't update ["
+ dvType + dvType
@ -676,7 +676,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
dvType, dvType,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,
@ -797,7 +797,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
fi.hasPayloads(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getIndexOptions(),
fi.getDocValuesType(), fi.getDocValuesType(),
fi.hasDocValuesSkipIndex(), fi.docValuesSkipIndexType(),
dvGen, dvGen,
// original attributes is UnmodifiableMap // original attributes is UnmodifiableMap
new HashMap<>(fi.attributes()), new HashMap<>(fi.attributes()),

View File

@ -87,7 +87,7 @@ public interface IndexableFieldType {
DocValuesType docValuesType(); DocValuesType docValuesType();
/** Whether a skip index for doc values should be created on this field. */ /** Whether a skip index for doc values should be created on this field. */
boolean hasDocValuesSkipIndex(); DocValuesSkipIndexType docValuesSkipIndexType();
/** /**
* If this is positive (representing the number of point dimensions), the field is indexed as a * If this is positive (representing the number of point dimensions), the field is indexed as a

View File

@ -680,7 +680,7 @@ final class IndexingChain implements Accountable {
false, false,
s.indexOptions, s.indexOptions,
s.docValuesType, s.docValuesType,
s.hasDocValuesSkipIndex, s.docValuesSkipIndex,
-1, -1,
s.attributes, s.attributes,
s.pointDimensionCount, s.pointDimensionCount,
@ -832,12 +832,14 @@ final class IndexingChain implements Accountable {
verifyUnIndexedFieldType(fieldName, fieldType); verifyUnIndexedFieldType(fieldName, fieldType);
} }
if (fieldType.docValuesType() != DocValuesType.NONE) { if (fieldType.docValuesType() != DocValuesType.NONE) {
schema.setDocValues(fieldType.docValuesType(), fieldType.hasDocValuesSkipIndex()); schema.setDocValues(fieldType.docValuesType(), fieldType.docValuesSkipIndexType());
} else if (fieldType.hasDocValuesSkipIndex()) { } else if (fieldType.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"field '" "field '"
+ schema.name + schema.name
+ "' cannot have docValuesSkipIndex set to true without doc values"); + "' cannot have docValuesSkipIndexType="
+ fieldType.docValuesSkipIndexType()
+ " without doc values");
} }
if (fieldType.pointDimensionCount() != 0) { if (fieldType.pointDimensionCount() != 0) {
schema.setPoints( schema.setPoints(
@ -1440,7 +1442,7 @@ final class IndexingChain implements Accountable {
private boolean storeTermVector = false; private boolean storeTermVector = false;
private IndexOptions indexOptions = IndexOptions.NONE; private IndexOptions indexOptions = IndexOptions.NONE;
private DocValuesType docValuesType = DocValuesType.NONE; private DocValuesType docValuesType = DocValuesType.NONE;
private boolean hasDocValuesSkipIndex = false; private DocValuesSkipIndexType docValuesSkipIndex = DocValuesSkipIndexType.NONE;
private int pointDimensionCount = 0; private int pointDimensionCount = 0;
private int pointIndexDimensionCount = 0; private int pointIndexDimensionCount = 0;
private int pointNumBytes = 0; private int pointNumBytes = 0;
@ -1506,13 +1508,14 @@ final class IndexingChain implements Accountable {
} }
} }
void setDocValues(DocValuesType newDocValuesType, boolean newHasDocValuesSkipIndex) { void setDocValues(
DocValuesType newDocValuesType, DocValuesSkipIndexType newDocValuesSkipIndex) {
if (docValuesType == DocValuesType.NONE) { if (docValuesType == DocValuesType.NONE) {
this.docValuesType = newDocValuesType; this.docValuesType = newDocValuesType;
this.hasDocValuesSkipIndex = newHasDocValuesSkipIndex; this.docValuesSkipIndex = newDocValuesSkipIndex;
} else { } else {
assertSame("doc values type", docValuesType, newDocValuesType); assertSame("doc values type", docValuesType, newDocValuesType);
assertSame("doc values skip index", hasDocValuesSkipIndex, newHasDocValuesSkipIndex); assertSame("doc values skip index type", docValuesSkipIndex, newDocValuesSkipIndex);
} }
} }
@ -1560,7 +1563,7 @@ final class IndexingChain implements Accountable {
assertSame("omit norms", fi.omitsNorms(), omitNorms); assertSame("omit norms", fi.omitsNorms(), omitNorms);
assertSame("store term vector", fi.hasTermVectors(), storeTermVector); assertSame("store term vector", fi.hasTermVectors(), storeTermVector);
assertSame("doc values type", fi.getDocValuesType(), docValuesType); assertSame("doc values type", fi.getDocValuesType(), docValuesType);
assertSame("doc values skip index", fi.hasDocValuesSkipIndex(), hasDocValuesSkipIndex); assertSame("doc values skip index type", fi.docValuesSkipIndexType(), docValuesSkipIndex);
assertSame( assertSame(
"vector similarity function", fi.getVectorSimilarityFunction(), vectorSimilarityFunction); "vector similarity function", fi.getVectorSimilarityFunction(), vectorSimilarityFunction);
assertSame("vector encoding", fi.getVectorEncoding(), vectorEncoding); assertSame("vector encoding", fi.getVectorEncoding(), vectorEncoding);

View File

@ -713,7 +713,7 @@ final class ReadersAndUpdates {
fi.hasPayloads(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getIndexOptions(),
fi.getDocValuesType(), fi.getDocValuesType(),
fi.hasDocValuesSkipIndex(), fi.docValuesSkipIndexType(),
fi.getDocValuesGen(), fi.getDocValuesGen(),
new HashMap<>(fi.attributes()), new HashMap<>(fi.attributes()),
fi.getPointDimensionCount(), fi.getPointDimensionCount(),

View File

@ -106,7 +106,7 @@ public class TestCodecs extends LuceneTestCase {
storePayloads, storePayloads,
indexOptions, indexOptions,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,

View File

@ -250,7 +250,7 @@ public class TestFieldInfos extends LuceneTestCase {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,
@ -272,7 +272,7 @@ public class TestFieldInfos extends LuceneTestCase {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,
@ -296,7 +296,7 @@ public class TestFieldInfos extends LuceneTestCase {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,

View File

@ -58,7 +58,7 @@ public class TestFieldsReader extends LuceneTestCase {
false, false,
ift.indexOptions(), ift.indexOptions(),
ift.docValuesType(), ift.docValuesType(),
ift.hasDocValuesSkipIndex(), ift.docValuesSkipIndexType(),
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,

View File

@ -4992,8 +4992,9 @@ public class TestIndexWriter extends LuceneTestCase {
doc2.add(new SortedNumericDocValuesField("test", random().nextLong())); doc2.add(new SortedNumericDocValuesField("test", random().nextLong()));
IllegalArgumentException ex = IllegalArgumentException ex =
expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc2)); expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc2));
ex.printStackTrace();
assertEquals( assertEquals(
"Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index: expected 'true', but it has 'false'.", "Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index type: expected 'RANGE', but it has 'NONE'.",
ex.getMessage()); ex.getMessage());
} }
} }
@ -5009,7 +5010,7 @@ public class TestIndexWriter extends LuceneTestCase {
IllegalArgumentException ex = IllegalArgumentException ex =
expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc2)); expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc2));
assertEquals( assertEquals(
"Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index: expected 'false', but it has 'true'.", "Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index type: expected 'NONE', but it has 'RANGE'.",
ex.getMessage()); ex.getMessage());
} }
} }
@ -5021,7 +5022,7 @@ public class TestIndexWriter extends LuceneTestCase {
FieldType fieldType = new FieldType(); FieldType fieldType = new FieldType();
fieldType.setStored(true); fieldType.setStored(true);
fieldType.setDocValuesType(docValuesType); fieldType.setDocValuesType(docValuesType);
fieldType.setDocValuesSkipIndex(true); fieldType.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE);
fieldType.freeze(); fieldType.freeze();
try (Directory dir = newMockDirectory()) { try (Directory dir = newMockDirectory()) {
try (IndexWriter writer = try (IndexWriter writer =
@ -5031,8 +5032,7 @@ public class TestIndexWriter extends LuceneTestCase {
IllegalArgumentException ex = IllegalArgumentException ex =
expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc1)); expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc1));
assertTrue( assertTrue(
ex.getMessage() ex.getMessage().startsWith("field 'test' cannot have docValuesSkipIndexType=RANGE"));
.startsWith("field 'test' cannot have docValuesSkipIndex set to true"));
} }
} }
} }

View File

@ -96,8 +96,8 @@ public class TestIndexableField extends LuceneTestCase {
} }
@Override @Override
public boolean hasDocValuesSkipIndex() { public DocValuesSkipIndexType docValuesSkipIndexType() {
return false; return DocValuesSkipIndexType.NONE;
} }
@Override @Override

View File

@ -191,7 +191,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NUMERIC, DocValuesType.NUMERIC,
false, DocValuesSkipIndexType.NONE,
0, 0,
Collections.emptyMap(), Collections.emptyMap(),
0, 0,
@ -231,7 +231,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NUMERIC, DocValuesType.NUMERIC,
false, DocValuesSkipIndexType.NONE,
1, 1,
Collections.emptyMap(), Collections.emptyMap(),
0, 0,
@ -297,7 +297,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NUMERIC, DocValuesType.NUMERIC,
false, DocValuesSkipIndexType.NONE,
segmentInfo.getNextDocValuesGen(), segmentInfo.getNextDocValuesGen(),
Collections.emptyMap(), Collections.emptyMap(),
0, 0,
@ -368,7 +368,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NUMERIC, DocValuesType.NUMERIC,
false, DocValuesSkipIndexType.NONE,
segmentInfo.getNextDocValuesGen(), segmentInfo.getNextDocValuesGen(),
Collections.emptyMap(), Collections.emptyMap(),
0, 0,
@ -407,7 +407,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NUMERIC, DocValuesType.NUMERIC,
false, DocValuesSkipIndexType.NONE,
segmentInfo.getNextDocValuesGen(), segmentInfo.getNextDocValuesGen(),
Collections.emptyMap(), Collections.emptyMap(),
0, 0,

View File

@ -1313,7 +1313,7 @@ public class TestSortOptimization extends LuceneTestCase {
false, false,
IndexOptions.NONE, IndexOptions.NONE,
fi.getDocValuesType(), fi.getDocValuesType(),
fi.hasDocValuesSkipIndex(), fi.docValuesSkipIndexType(),
fi.getDocValuesGen(), fi.getDocValuesGen(),
fi.attributes(), fi.attributes(),
0, 0,

View File

@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.Iterator; import java.util.Iterator;
import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
@ -96,7 +97,7 @@ public class TermVectorLeafReader extends LeafReader {
terms.hasPayloads(), terms.hasPayloads(),
indexOptions, indexOptions,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
Collections.emptyMap(), Collections.emptyMap(),
0, 0,

View File

@ -736,7 +736,7 @@ public class MemoryIndex {
storePayloads, storePayloads,
indexOptions, indexOptions,
fieldType.docValuesType(), fieldType.docValuesType(),
false, fieldType.docValuesSkipIndexType(),
-1, -1,
Collections.emptyMap(), Collections.emptyMap(),
fieldType.pointDimensionCount(), fieldType.pointDimensionCount(),
@ -841,7 +841,7 @@ public class MemoryIndex {
info.fieldInfo.hasPayloads(), info.fieldInfo.hasPayloads(),
info.fieldInfo.getIndexOptions(), info.fieldInfo.getIndexOptions(),
docValuesType, docValuesType,
false, DocValuesSkipIndexType.NONE,
-1, -1,
info.fieldInfo.attributes(), info.fieldInfo.attributes(),
info.fieldInfo.getPointDimensionCount(), info.fieldInfo.getPointDimensionCount(),

View File

@ -23,6 +23,7 @@ import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
@ -283,7 +284,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
@Override @Override
public DocValuesSkipper getSkipper(FieldInfo field) throws IOException { public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
assert field.hasDocValuesSkipIndex(); assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE;
DocValuesSkipper skipper = in.getSkipper(field); DocValuesSkipper skipper = in.getSkipper(field);
assert skipper != null; assert skipper != null;
return new AssertingLeafReader.AssertingDocValuesSkipper(skipper); return new AssertingLeafReader.AssertingDocValuesSkipper(skipper);

View File

@ -23,6 +23,7 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
@ -1625,10 +1626,10 @@ public class AssertingLeafReader extends FilterLeafReader {
DocValuesSkipper skipper = super.getDocValuesSkipper(field); DocValuesSkipper skipper = super.getDocValuesSkipper(field);
FieldInfo fi = getFieldInfos().fieldInfo(field); FieldInfo fi = getFieldInfos().fieldInfo(field);
if (skipper != null) { if (skipper != null) {
assert fi.hasDocValuesSkipIndex(); assert fi.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE;
return new AssertingDocValuesSkipper(skipper); return new AssertingDocValuesSkipper(skipper);
} else { } else {
assert fi == null || fi.hasDocValuesSkipIndex() == false; assert fi == null || fi.docValuesSkipIndexType() == DocValuesSkipIndexType.NONE;
return null; return null;
} }
} }

View File

@ -30,6 +30,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField; import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -303,14 +304,14 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
storePayloads = random().nextBoolean(); storePayloads = random().nextBoolean();
} }
} }
boolean hasDocValuesSkipIndex = false; DocValuesSkipIndexType docValuesSkipIndexType = DocValuesSkipIndexType.NONE;
if (EnumSet.of( if (EnumSet.of(
DocValuesType.NUMERIC, DocValuesType.NUMERIC,
DocValuesType.SORTED, DocValuesType.SORTED,
DocValuesType.SORTED_NUMERIC, DocValuesType.SORTED_NUMERIC,
DocValuesType.SORTED_SET) DocValuesType.SORTED_SET)
.contains(fieldType.docValuesType())) { .contains(fieldType.docValuesType())) {
hasDocValuesSkipIndex = fieldType.hasDocValuesSkipIndex(); docValuesSkipIndexType = fieldType.docValuesSkipIndexType();
} }
FieldInfo fi = FieldInfo fi =
new FieldInfo( new FieldInfo(
@ -321,7 +322,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
storePayloads, storePayloads,
fieldType.indexOptions(), fieldType.indexOptions(),
fieldType.docValuesType(), fieldType.docValuesType(),
hasDocValuesSkipIndex, docValuesSkipIndexType,
-1, -1,
new HashMap<>(), new HashMap<>(),
fieldType.pointDimensionCount(), fieldType.pointDimensionCount(),
@ -374,7 +375,10 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
|| current == DocValuesType.SORTED_NUMERIC || current == DocValuesType.SORTED_NUMERIC
|| current == DocValuesType.SORTED || current == DocValuesType.SORTED
|| current == DocValuesType.SORTED_SET) { || current == DocValuesType.SORTED_SET) {
type.setDocValuesSkipIndex(supportDocValuesSkipIndex() && random().nextBoolean()); type.setDocValuesSkipIndexType(
supportDocValuesSkipIndex()
? DocValuesSkipIndexType.RANGE
: DocValuesSkipIndexType.NONE);
} }
} }
@ -414,7 +418,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
assertEquals(expected.number, actual.number); assertEquals(expected.number, actual.number);
assertEquals(expected.name, actual.name); assertEquals(expected.name, actual.name);
assertEquals(expected.getDocValuesType(), actual.getDocValuesType()); assertEquals(expected.getDocValuesType(), actual.getDocValuesType());
assertEquals(expected.hasDocValuesSkipIndex(), actual.hasDocValuesSkipIndex()); assertEquals(expected.docValuesSkipIndexType(), actual.docValuesSkipIndexType());
assertEquals(expected.getIndexOptions(), actual.getIndexOptions()); assertEquals(expected.getIndexOptions(), actual.getIndexOptions());
assertEquals(expected.hasNorms(), actual.hasNorms()); assertEquals(expected.hasNorms(), actual.hasNorms());
assertEquals(expected.hasPayloads(), actual.hasPayloads()); assertEquals(expected.hasPayloads(), actual.hasPayloads());
@ -455,7 +459,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
false, false,
TextField.TYPE_STORED.indexOptions(), TextField.TYPE_STORED.indexOptions(),
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,

View File

@ -357,7 +357,7 @@ public abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
proto.hasPayloads(), proto.hasPayloads(),
proto.getIndexOptions(), proto.getIndexOptions(),
proto.getDocValuesType(), proto.getDocValuesType(),
proto.hasDocValuesSkipIndex(), proto.docValuesSkipIndexType(),
proto.getDocValuesGen(), proto.getDocValuesGen(),
new HashMap<>(), new HashMap<>(),
proto.getPointDimensionCount(), proto.getPointDimensionCount(),

View File

@ -46,6 +46,7 @@ import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.CodecReader; import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -377,7 +378,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
false, false,
IndexOptions.NONE, IndexOptions.NONE,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
Map.of(), Map.of(),
0, 0,

View File

@ -103,7 +103,7 @@ public class MismatchedLeafReader extends FilterLeafReader {
oldInfo.hasPayloads(), // storePayloads oldInfo.hasPayloads(), // storePayloads
oldInfo.getIndexOptions(), // indexOptions oldInfo.getIndexOptions(), // indexOptions
oldInfo.getDocValuesType(), // docValuesType oldInfo.getDocValuesType(), // docValuesType
oldInfo.hasDocValuesSkipIndex(), // hasDocValuesSkipIndex oldInfo.docValuesSkipIndexType(), // docValuesSkipIndexType
oldInfo.getDocValuesGen(), // dvGen oldInfo.getDocValuesGen(), // dvGen
oldInfo.attributes(), // attributes oldInfo.attributes(), // attributes
oldInfo.getPointDimensionCount(), // data dimension count oldInfo.getPointDimensionCount(), // data dimension count

View File

@ -45,6 +45,7 @@ import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.index.BaseTermsEnum; import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfos;
@ -157,7 +158,7 @@ public class RandomPostingsTester {
true, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,
@ -732,7 +733,7 @@ public class RandomPostingsTester {
doPayloads, doPayloads,
indexOptions, indexOptions,
DocValuesType.NONE, DocValuesType.NONE,
false, DocValuesSkipIndexType.NONE,
-1, -1,
new HashMap<>(), new HashMap<>(),
0, 0,