Modernize LineFileDocs. (#12929)

This replaces `StringField`/`SortedDocValuesField` with `KeywordField` and
`IntPoint`/`NumericDocValuesField` with `IntField`.
This commit is contained in:
Adrien Grand 2023-12-19 11:25:26 +01:00 committed by GitHub
parent 5c084fcd6e
commit bcc7e120ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 27 additions and 20 deletions

View File

@ -64,7 +64,7 @@ public class TestAllFilesCheckIndexHeader extends LuceneTestCase {
} }
if (random().nextInt(15) == 0) { if (random().nextInt(15) == 0) {
riw.updateNumericDocValue( riw.updateNumericDocValue(
new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); new Term("docid", Integer.toString(i)), "page_views", Long.valueOf(i));
} }
} }

View File

@ -46,7 +46,7 @@ public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
} }
if (random().nextInt(15) == 0) { if (random().nextInt(15) == 0) {
riw.updateNumericDocValue( riw.updateNumericDocValue(
new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); new Term("docid", Integer.toString(i)), "page_views", Long.valueOf(i));
} }
} }
riw.close(); riw.close();

View File

@ -49,7 +49,7 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
} }
if (random().nextInt(15) == 0) { if (random().nextInt(15) == 0) {
riw.updateNumericDocValue( riw.updateNumericDocValue(
new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); new Term("docid", Integer.toString(i)), "page_views", Long.valueOf(i));
} }
} }
riw.close(); riw.close();

View File

@ -37,14 +37,14 @@ import java.util.zip.GZIPInputStream;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType; import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.KeywordField;
import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField; import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField; import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CloseableThreadLocal; import org.apache.lucene.util.CloseableThreadLocal;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
@ -199,17 +199,16 @@ public class LineFileDocs implements Closeable {
final Document doc; final Document doc;
final Field titleTokenized; final Field titleTokenized;
final Field title; final Field title;
final Field titleDV;
final Field body; final Field body;
final Field id; final Field id;
final Field idNum; final Field idNum;
final Field idNumDV;
final Field date; final Field date;
final Field pageViews;
public DocState() { public DocState() {
doc = new Document(); doc = new Document();
title = new StringField("title", "", Field.Store.NO); title = new KeywordField("title", "", Field.Store.NO);
doc.add(title); doc.add(title);
FieldType ft = new FieldType(TextField.TYPE_STORED); FieldType ft = new FieldType(TextField.TYPE_STORED);
@ -227,16 +226,15 @@ public class LineFileDocs implements Closeable {
id = new StringField("docid", "", Field.Store.YES); id = new StringField("docid", "", Field.Store.YES);
doc.add(id); doc.add(id);
idNum = new IntPoint("docid_int", 0); idNum = new IntField("docid_int", 0, Field.Store.NO);
doc.add(idNum); doc.add(idNum);
date = new StringField("date", "", Field.Store.YES); date = new StringField("date", "", Field.Store.YES);
doc.add(date); doc.add(date);
titleDV = new SortedDocValuesField("titleDV", new BytesRef()); // A numeric DV field that can be used for DV updates
idNumDV = new NumericDocValuesField("docid_intDV", 0); pageViews = new NumericDocValuesField("page_views", 0L);
doc.add(titleDV); doc.add(pageViews);
doc.add(idNumDV);
} }
} }
@ -277,17 +275,12 @@ public class LineFileDocs implements Closeable {
docState.body.setStringValue(line.substring(1 + spot2, line.length())); docState.body.setStringValue(line.substring(1 + spot2, line.length()));
final String title = line.substring(0, spot); final String title = line.substring(0, spot);
docState.title.setStringValue(title); docState.title.setStringValue(title);
if (docState.titleDV != null) {
docState.titleDV.setBytesValue(new BytesRef(title));
}
docState.titleTokenized.setStringValue(title); docState.titleTokenized.setStringValue(title);
docState.date.setStringValue(line.substring(1 + spot, spot2)); docState.date.setStringValue(line.substring(1 + spot, spot2));
final int i = id.getAndIncrement(); final int i = id.getAndIncrement();
docState.id.setStringValue(Integer.toString(i)); docState.id.setStringValue(Integer.toString(i));
docState.idNum.setIntValue(i); docState.idNum.setIntValue(i);
if (docState.idNumDV != null) { docState.pageViews.setLongValue(random.nextInt(10_000));
docState.idNumDV.setLongValue(i);
}
if (random.nextInt(5) == 4) { if (random.nextInt(5) == 4) {
// Make some sparse fields // Make some sparse fields

View File

@ -63,6 +63,8 @@ import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.KeywordField;
import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.CheckIndex;
@ -1420,7 +1422,19 @@ public final class TestUtil {
final Field field2; final Field field2;
final DocValuesType dvType = field1.fieldType().docValuesType(); final DocValuesType dvType = field1.fieldType().docValuesType();
final int dimCount = field1.fieldType().pointDimensionCount(); final int dimCount = field1.fieldType().pointDimensionCount();
if (dvType != DocValuesType.NONE) { if (f instanceof KeywordField) {
field2 =
new KeywordField(
f.name(),
f.stringValue(),
f.fieldType().stored() ? Field.Store.YES : Field.Store.NO);
} else if (f instanceof IntField) {
field2 =
new IntField(
f.name(),
f.numericValue().intValue(),
f.fieldType().stored() ? Field.Store.YES : Field.Store.NO);
} else if (dvType != DocValuesType.NONE) {
switch (dvType) { switch (dvType) {
case NUMERIC: case NUMERIC:
field2 = new NumericDocValuesField(field1.name(), field1.numericValue().longValue()); field2 = new NumericDocValuesField(field1.name(), field1.numericValue().longValue());