Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8134: Index options are no longer automatically downgraded.

This commit is contained in:
Adrien Grand 2018-02-15 13:00:09 +01:00
parent c4c6b2a796
commit 890e8a51f8
15 changed files with 228 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -55,6 +55,9 @@ Changes in Runtime Behavior
* LUCENE-8116: Similarities now score fields that omit norms as if the norm was
1. This might change score values on fields that omit norms. (Adrien Grand)
* LUCENE-8134: Index options are no longer automatically downgraded.
(Adrien Grand)
Improvements
* LUCENE-7997: Add BaseSimilarityTestCase to sanity check similarities.

7
lucene/MIGRATE.txt
View File

@ -30,3 +30,10 @@ bindings.add("boost1", DoubleValuesSource.fromIntField("myboostfield"));
bindings.add("boost2", DoubleValuesSource.fromIntField("myotherboostfield"));
Expression expr = JavascriptCompiler.compile("score * (boost1 + ln(boost2))");
FunctionScoreQuery q = new FunctionScoreQuery(inputQuery, expr.getDoubleValuesSource(bindings));
## Index options can no longer be changed dynamically (LUCENE-8134) ##
Changing index options on the fly is now going to result into an
IllegalArgumentException. If a field is indexed
(FieldType.indexOptions() != IndexOptions.NONE) then all documents must have
the same index options for that field.

5
lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java
View File

@ -285,7 +285,7 @@ public class TestOrdsBlockTree extends BasePostingsFormatTestCase {
if (VERBOSE) {
System.out.println("i=" + i + " term=" + term);
}
doc.add(newTextField("field", term, Field.Store.NO));
doc.add(newStringField("field", term, Field.Store.NO));
w.addDocument(doc);
}
for(int i=0;i<128;i++) {
@ -373,7 +373,8 @@ public class TestOrdsBlockTree extends BasePostingsFormatTestCase {
doc = new Document();
String term = "" + (char) (97+i);
String term2 = "a" + (char) (97+i);
doc.add(newTextField("field", term + " " + term2, Field.Store.NO));
doc.add(newStringField("field", term, Field.Store.NO));
doc.add(newStringField("field", term2, Field.Store.NO));
w.addDocument(doc);
}

22
lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
View File

@ -608,11 +608,7 @@ final class DefaultIndexingChain extends DocConsumer {
// First time we are seeing this field in this segment
FieldInfo fi = fieldInfos.getOrAdd(name);
// Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial
// IndexOptions to decide what arrays it must create). Then, we also must set it in
// PerField.invert to allow for later downgrading of the index options:
fi.setIndexOptions(fieldType.indexOptions());
initIndexOptions(fi, fieldType.indexOptions());
fp = new PerField(docWriter.getIndexCreatedVersionMajor(), fi, invert);
fp.next = fieldHash[hashPos];
fieldHash[hashPos] = fp;
@ -630,16 +626,24 @@ final class DefaultIndexingChain extends DocConsumer {
}
} else if (invert && fp.invertState == null) {
// Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial
// IndexOptions to decide what arrays it must create). Then, we also must set it in
// PerField.invert to allow for later downgrading of the index options:
fp.fieldInfo.setIndexOptions(fieldType.indexOptions());
initIndexOptions(fp.fieldInfo, fieldType.indexOptions());
fp.setInvertState();
}
return fp;
}
private void initIndexOptions(FieldInfo info, IndexOptions indexOptions) {
// Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial
// IndexOptions to decide what arrays it must create).
assert info.getIndexOptions() == IndexOptions.NONE;
info.setIndexOptions(indexOptions);
// This is the first time we are seeing this field indexed, so we now
// record the index options so that any future attempt to (illegally)
// change the index options of this field, will throw an IllegalArgExc:
fieldInfos.globalFieldNumbers.setIndexOptions(info.number, info.name, indexOptions);
}
/** NOTE: not static: accesses at least docState, termsHash. */
private final class PerField implements Comparable<PerField> {

6
lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
View File

@ -137,8 +137,7 @@ public final class FieldInfo {
if (this.indexOptions == IndexOptions.NONE) {
this.indexOptions = indexOptions;
} else if (indexOptions != IndexOptions.NONE) {
// downgrade
this.indexOptions = this.indexOptions.compareTo(indexOptions) < 0 ? this.indexOptions : indexOptions;
throw new IllegalArgumentException("cannot change field \"" + name + "\" from index options=" + this.indexOptions + " to inconsistent index options=" + indexOptions);
}
}
@ -226,8 +225,7 @@ public final class FieldInfo {
if (indexOptions == IndexOptions.NONE) {
indexOptions = newIndexOptions;
} else if (newIndexOptions != IndexOptions.NONE) {
// downgrade
indexOptions = indexOptions.compareTo(newIndexOptions) < 0 ? indexOptions : newIndexOptions;
throw new IllegalArgumentException("cannot change field \"" + name + "\" from index options=" + indexOptions + " to inconsistent index options=" + newIndexOptions);
}
}

35
lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
View File

@ -209,6 +209,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
private final Map<Integer,String> numberToName;
private final Map<String,Integer> nameToNumber;
private final Map<String, IndexOptions> indexOptions;
// We use this to enforce that a given field never
// changes DV type, even across segments / IndexWriter
// sessions:
@ -224,6 +225,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
FieldNumbers() {
this.nameToNumber = new HashMap<>();
this.numberToName = new HashMap<>();
this.indexOptions = new HashMap<>();
this.docValuesType = new HashMap<>();
this.dimensions = new HashMap<>();
}
@ -234,7 +236,15 @@ public class FieldInfos implements Iterable<FieldInfo> {
* number assigned if possible otherwise the first unassigned field number
* is used as the field number.
*/
synchronized int addOrGet(String fieldName, int preferredFieldNumber, DocValuesType dvType, int dimensionCount, int dimensionNumBytes) {
synchronized int addOrGet(String fieldName, int preferredFieldNumber, IndexOptions indexOptions, DocValuesType dvType, int dimensionCount, int dimensionNumBytes) {
if (indexOptions != IndexOptions.NONE) {
IndexOptions currentOpts = this.indexOptions.get(fieldName);
if (currentOpts == null) {
this.indexOptions.put(fieldName, indexOptions);
} else if (currentOpts != IndexOptions.NONE && currentOpts != indexOptions) {
throw new IllegalArgumentException("cannot change field \"" + fieldName + "\" from index options=" + currentOpts + " to inconsistent index options=" + indexOptions);
}
}
if (dvType != DocValuesType.NONE) {
DocValuesType currentDVType = docValuesType.get(fieldName);
if (currentDVType == null) {
@ -277,6 +287,19 @@ public class FieldInfos implements Iterable<FieldInfo> {
return fieldNumber.intValue();
}
synchronized void verifyConsistent(Integer number, String name, IndexOptions indexOptions) {
if (name.equals(numberToName.get(number)) == false) {
throw new IllegalArgumentException("field number " + number + " is already mapped to field name \"" + numberToName.get(number) + "\", not \"" + name + "\"");
}
if (number.equals(nameToNumber.get(name)) == false) {
throw new IllegalArgumentException("field name \"" + name + "\" is already mapped to field number \"" + nameToNumber.get(name) + "\", not \"" + number + "\"");
}
IndexOptions currentIndexOptions = this.indexOptions.get(name);
if (indexOptions != IndexOptions.NONE && currentIndexOptions != null && currentIndexOptions != IndexOptions.NONE && indexOptions != currentIndexOptions) {
throw new IllegalArgumentException("cannot change index options from " + currentIndexOptions + " to " + indexOptions + " for field \"" + name + "\"");
}
}
synchronized void verifyConsistent(Integer number, String name, DocValuesType dvType) {
if (name.equals(numberToName.get(number)) == false) {
throw new IllegalArgumentException("field number " + number + " is already mapped to field name \"" + numberToName.get(number) + "\", not \"" + name + "\"");
@ -329,10 +352,16 @@ public class FieldInfos implements Iterable<FieldInfo> {
synchronized void clear() {
numberToName.clear();
nameToNumber.clear();
indexOptions.clear();
docValuesType.clear();
dimensions.clear();
}
synchronized void setIndexOptions(int number, String name, IndexOptions indexOptions) {
verifyConsistent(number, name, indexOptions);
this.indexOptions.put(name, indexOptions);
}
synchronized void setDocValuesType(int number, String name, DocValuesType dvType) {
verifyConsistent(number, name, dvType);
docValuesType.put(name, dvType);
@ -384,7 +413,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
// number for this field. If the field was seen
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, DocValuesType.NONE, 0, 0);
final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, IndexOptions.NONE, DocValuesType.NONE, 0, 0);
fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, new HashMap<>(), 0, 0);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, DocValuesType.NONE);
@ -410,7 +439,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
// number for this field. If the field was seen
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues, dimensionCount, dimensionNumBytes);
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, indexOptions, docValues, dimensionCount, dimensionNumBytes);
fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());

4
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -1219,7 +1219,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
for(SegmentCommitInfo info : segmentInfos) {
FieldInfos fis = readFieldInfos(info);
for(FieldInfo fi : fis) {
map.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes());
map.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes());
}
}
@ -2878,7 +2878,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
FieldInfos fis = readFieldInfos(info);
for(FieldInfo fi : fis) {
// This will throw exceptions if any of the incoming fields have an illegal schema change:
globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes());
globalFieldNumberMap.addOrGet(fi.name, fi.number, fi.getIndexOptions(), fi.getDocValuesType(), fi.getPointDimensionCount(), fi.getPointNumBytes());
}
infos.add(copySegmentAsIs(info, newSegName, context));
}

5
lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
View File

@ -24,7 +24,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FailOnNonBulkMergesInfoStream;
@ -41,8 +40,8 @@ public class TestConsistentFieldNumbers extends LuceneTestCase {
.setMergePolicy(NoMergePolicy.INSTANCE));
Document d1 = new Document();
d1.add(new StringField("f1", "first field", Field.Store.YES));
d1.add(new StringField("f2", "second field", Field.Store.YES));
d1.add(new TextField("f1", "first field", Field.Store.YES));
d1.add(new TextField("f2", "second field", Field.Store.YES));
writer.addDocument(d1);
if (i == 1) {

148
lucene/core/src/test/org/apache/lucene/index/TestIndexOptions.java Normal file
View File

@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collections;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
public class TestIndexOptions extends LuceneTestCase {
public void testChangeIndexOptionsViaAddDocument() throws IOException {
for (IndexOptions from : IndexOptions.values()) {
for (IndexOptions to : IndexOptions.values()) {
for (boolean preExisting : new boolean[] { false, true }) {
doTestChangeIndexOptionsViaAddDocument(preExisting, from, to);
}
}
}
}
private void doTestChangeIndexOptionsViaAddDocument(boolean preExistingField, IndexOptions from, IndexOptions to) throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
if (preExistingField) {
w.addDocument(Collections.singleton(new IntPoint("foo", 1)));
}
FieldType ft1 = new FieldType(TextField.TYPE_STORED);
ft1.setIndexOptions(from);
w.addDocument(Collections.singleton(new Field("foo", "bar", ft1)));
FieldType ft2 = new FieldType(TextField.TYPE_STORED);
ft2.setIndexOptions(to);
if (from == IndexOptions.NONE || to == IndexOptions.NONE || from == to) {
w.addDocument(Collections.singleton(new Field("foo", "bar", ft2))); // no exception
w.forceMerge(1);
try (LeafReader r = getOnlyLeafReader(DirectoryReader.open(w))) {
IndexOptions expected = from == IndexOptions.NONE ? to : from;
assertEquals(expected, r.getFieldInfos().fieldInfo("foo").getIndexOptions());
}
} else {
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> w.addDocument(Collections.singleton(new Field("foo", "bar", ft2))));
assertEquals("cannot change field \"foo\" from index options=" + from +
" to inconsistent index options=" + to, e.getMessage());
}
w.close();
dir.close();
}
public void testChangeIndexOptionsViaAddIndexesCodecReader() throws IOException {
for (IndexOptions from : IndexOptions.values()) {
for (IndexOptions to : IndexOptions.values()) {
doTestChangeIndexOptionsAddIndexesCodecReader(from, to);
}
}
}
private void doTestChangeIndexOptionsAddIndexesCodecReader(IndexOptions from, IndexOptions to) throws IOException {
Directory dir1 = newDirectory();
IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig());
FieldType ft1 = new FieldType(TextField.TYPE_STORED);
ft1.setIndexOptions(from);
w1.addDocument(Collections.singleton(new Field("foo", "bar", ft1)));
Directory dir2 = newDirectory();
IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig());
FieldType ft2 = new FieldType(TextField.TYPE_STORED);
ft2.setIndexOptions(to);
w2.addDocument(Collections.singleton(new Field("foo", "bar", ft2)));
try (CodecReader cr = (CodecReader) getOnlyLeafReader(DirectoryReader.open(w2))) {
if (from == IndexOptions.NONE || to == IndexOptions.NONE || from == to) {
w1.addIndexes(cr); // no exception
w1.forceMerge(1);
try (LeafReader r = getOnlyLeafReader(DirectoryReader.open(w1))) {
IndexOptions expected = from == IndexOptions.NONE ? to : from;
assertEquals(expected, r.getFieldInfos().fieldInfo("foo").getIndexOptions());
}
} else {
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> w1.addIndexes(cr));
assertEquals("cannot change field \"foo\" from index options=" + from +
" to inconsistent index options=" + to, e.getMessage());
}
}
IOUtils.close(w1, w2, dir1, dir2);
}
public void testChangeIndexOptionsViaAddIndexesDirectory() throws IOException {
for (IndexOptions from : IndexOptions.values()) {
for (IndexOptions to : IndexOptions.values()) {
doTestChangeIndexOptionsAddIndexesDirectory(from, to);
}
}
}
private void doTestChangeIndexOptionsAddIndexesDirectory(IndexOptions from, IndexOptions to) throws IOException {
Directory dir1 = newDirectory();
IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig());
FieldType ft1 = new FieldType(TextField.TYPE_STORED);
ft1.setIndexOptions(from);
w1.addDocument(Collections.singleton(new Field("foo", "bar", ft1)));
Directory dir2 = newDirectory();
IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig());
FieldType ft2 = new FieldType(TextField.TYPE_STORED);
ft2.setIndexOptions(to);
w2.addDocument(Collections.singleton(new Field("foo", "bar", ft2)));
w2.close();
if (from == IndexOptions.NONE || to == IndexOptions.NONE || from == to) {
w1.addIndexes(dir2); // no exception
w1.forceMerge(1);
try (LeafReader r = getOnlyLeafReader(DirectoryReader.open(w1))) {
IndexOptions expected = from == IndexOptions.NONE ? to : from;
assertEquals(expected, r.getFieldInfos().fieldInfo("foo").getIndexOptions());
}
} else {
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> w1.addIndexes(dir2));
assertEquals("cannot change field \"foo\" from index options=" + from +
" to inconsistent index options=" + to, e.getMessage());
}
IOUtils.close(w1, dir1, dir2);
}
}

23
lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -1627,29 +1627,6 @@ public class TestIndexWriter extends LuceneTestCase {
d.close();
}
public void testChangeIndexOptions() throws Exception {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir,
new IndexWriterConfig(new MockAnalyzer(random())));
FieldType docsAndFreqs = new FieldType(TextField.TYPE_NOT_STORED);
docsAndFreqs.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
FieldType docsOnly = new FieldType(TextField.TYPE_NOT_STORED);
docsOnly.setIndexOptions(IndexOptions.DOCS);
Document doc = new Document();
doc.add(new Field("field", "a b c", docsAndFreqs));
w.addDocument(doc);
w.addDocument(doc);
doc = new Document();
doc.add(new Field("field", "a b c", docsOnly));
w.addDocument(doc);
w.close();
dir.close();
}
public void testOnlyUpdateDocuments() throws Exception {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir,

138
lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
View File

@ -69,86 +69,29 @@ public class TestOmitPositions extends LuceneTestCase {
IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer));
Document d = new Document();
// f1,f2,f3: docs only
// f1: docs only
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.setIndexOptions(IndexOptions.DOCS);
Field f1 = newField("f1", "This field has docs only", ft);
d.add(f1);
Field f2 = newField("f2", "This field has docs only", ft);
d.add(f2);
Field f3 = newField("f3", "This field has docs only", ft);
d.add(f3);
FieldType ft2 = new FieldType(TextField.TYPE_NOT_STORED);
ft2.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
// f4,f5,f6 docs and freqs
Field f4 = newField("f4", "This field has docs and freqs", ft2);
d.add(f4);
Field f5 = newField("f5", "This field has docs and freqs", ft2);
d.add(f5);
Field f6 = newField("f6", "This field has docs and freqs", ft2);
d.add(f6);
// f2: docs and freqs
Field f2 = newField("f2", "This field has docs and freqs", ft2);
d.add(f2);
FieldType ft3 = new FieldType(TextField.TYPE_NOT_STORED);
ft3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
// f7,f8,f9 docs/freqs/positions
Field f7 = newField("f7", "This field has docs and freqs and positions", ft3);
d.add(f7);
Field f8 = newField("f8", "This field has docs and freqs and positions", ft3);
d.add(f8);
Field f9 = newField("f9", "This field has docs and freqs and positions", ft3);
d.add(f9);
// f3: docs/freqs/positions
Field f3 = newField("f3", "This field has docs and freqs and positions", ft3);
d.add(f3);
writer.addDocument(d);
writer.forceMerge(1);
// now we add another document which has docs-only for f1, f4, f7, docs/freqs for f2, f5, f8,
// and docs/freqs/positions for f3, f6, f9
d = new Document();
// f1,f4,f7: docs only
f1 = newField("f1", "This field has docs only", ft);
d.add(f1);
f4 = newField("f4", "This field has docs only", ft);
d.add(f4);
f7 = newField("f7", "This field has docs only", ft);
d.add(f7);
// f2, f5, f8: docs and freqs
f2 = newField("f2", "This field has docs and freqs", ft2);
d.add(f2);
f5 = newField("f5", "This field has docs and freqs", ft2);
d.add(f5);
f8 = newField("f8", "This field has docs and freqs", ft2);
d.add(f8);
// f3, f6, f9: docs and freqs and positions
f3 = newField("f3", "This field has docs and freqs and positions", ft3);
d.add(f3);
f6 = newField("f6", "This field has docs and freqs and positions", ft3);
d.add(f6);
f9 = newField("f9", "This field has docs and freqs and positions", ft3);
d.add(f9);
writer.addDocument(d);
// force merge
writer.forceMerge(1);
// flush
writer.close();
@ -156,22 +99,10 @@ public class TestOmitPositions extends LuceneTestCase {
FieldInfos fi = reader.getFieldInfos();
// docs + docs = docs
assertEquals(IndexOptions.DOCS, fi.fieldInfo("f1").getIndexOptions());
// docs + docs/freqs = docs
assertEquals(IndexOptions.DOCS, fi.fieldInfo("f2").getIndexOptions());
// docs + docs/freqs/pos = docs
assertEquals(IndexOptions.DOCS, fi.fieldInfo("f3").getIndexOptions());
// docs/freqs + docs = docs
assertEquals(IndexOptions.DOCS, fi.fieldInfo("f4").getIndexOptions());
// docs/freqs + docs/freqs = docs/freqs
assertEquals(IndexOptions.DOCS_AND_FREQS, fi.fieldInfo("f5").getIndexOptions());
// docs/freqs + docs/freqs/pos = docs/freqs
assertEquals(IndexOptions.DOCS_AND_FREQS, fi.fieldInfo("f6").getIndexOptions());
// docs/freqs/pos + docs = docs
assertEquals(IndexOptions.DOCS, fi.fieldInfo("f7").getIndexOptions());
// docs/freqs/pos + docs/freqs = docs/freqs
assertEquals(IndexOptions.DOCS_AND_FREQS, fi.fieldInfo("f8").getIndexOptions());
assertEquals(IndexOptions.DOCS_AND_FREQS, fi.fieldInfo("f2").getIndexOptions());
// docs/freqs/pos + docs/freqs/pos = docs/freqs/pos
assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f9").getIndexOptions());
assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f3").getIndexOptions());
reader.close();
ram.close();
@ -210,57 +141,8 @@ public class TestOmitPositions extends LuceneTestCase {
assertNoPrx(ram);
// now add some documents with positions, and check there is no prox after optimization
d = new Document();
f1 = newTextField("f1", "This field has positions", Field.Store.NO);
d.add(f1);
for(int i=0;i<30;i++)
writer.addDocument(d);
// force merge
writer.forceMerge(1);
// flush
writer.close();
assertNoPrx(ram);
ram.close();
}
/** make sure we downgrade positions and payloads correctly */
public void testMixing() throws Exception {
// no positions
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
for (int i = 0; i < 20; i++) {
Document doc = new Document();
if (i < 19 && random().nextBoolean()) {
for (int j = 0; j < 50; j++) {
doc.add(new TextField("foo", "i have positions", Field.Store.NO));
}
} else {
for (int j = 0; j < 50; j++) {
doc.add(new Field("foo", "i have no positions", ft));
}
}
iw.addDocument(doc);
iw.commit();
}
if (random().nextBoolean()) {
iw.forceMerge(1);
}
DirectoryReader ir = iw.getReader();
FieldInfos fis = MultiFields.getMergedFieldInfos(ir);
assertEquals(IndexOptions.DOCS_AND_FREQS, fis.fieldInfo("foo").getIndexOptions());
assertFalse(fis.fieldInfo("foo").hasPayloads());
iw.close();
ir.close();
dir.close(); // checkindex
}
}

111
lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
View File

@ -60,103 +60,6 @@ public class TestOmitTf extends LuceneTestCase {
omitType.setIndexOptions(IndexOptions.DOCS);
}
// Tests whether the DocumentWriter correctly enable the
// omitTermFreqAndPositions bit in the FieldInfo
public void testOmitTermFreqAndPositions() throws Exception {
Directory ram = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer));
Document d = new Document();
// this field will have Tf
Field f1 = newField("f1", "This field has term freqs", normalType);
d.add(f1);
// this field will NOT have Tf
Field f2 = newField("f2", "This field has NO Tf in all docs", omitType);
d.add(f2);
writer.addDocument(d);
writer.forceMerge(1);
// now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
// keep things constant
d = new Document();
// Reverse
f1 = newField("f1", "This field has term freqs", omitType);
d.add(f1);
f2 = newField("f2", "This field has NO Tf in all docs", normalType);
d.add(f2);
writer.addDocument(d);
// force merge
writer.forceMerge(1);
// flush
writer.close();
LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram));
FieldInfos fi = reader.getFieldInfos();
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f1").getIndexOptions());
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f2").getIndexOptions());
reader.close();
ram.close();
}
// Tests whether merging of docs that have different
// omitTermFreqAndPositions for the same field works
public void testMixedMerge() throws Exception {
Directory ram = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer = new IndexWriter(
ram,
newIndexWriterConfig(analyzer).
setMaxBufferedDocs(3).
setMergePolicy(newLogMergePolicy(2))
);
Document d = new Document();
// this field will have Tf
Field f1 = newField("f1", "This field has term freqs", normalType);
d.add(f1);
// this field will NOT have Tf
Field f2 = newField("f2", "This field has NO Tf in all docs", omitType);
d.add(f2);
for(int i=0;i<30;i++)
writer.addDocument(d);
// now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
// keep things constant
d = new Document();
// Reverese
f1 = newField("f1", "This field has term freqs", omitType);
d.add(f1);
f2 = newField("f2", "This field has NO Tf in all docs", normalType);
d.add(f2);
for(int i=0;i<30;i++)
writer.addDocument(d);
// force merge
writer.forceMerge(1);
// flush
writer.close();
LeafReader reader = getOnlyLeafReader(DirectoryReader.open(ram));
FieldInfos fi = reader.getFieldInfos();
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f1").getIndexOptions());
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS, fi.fieldInfo("f2").getIndexOptions());
reader.close();
ram.close();
}
// Make sure first adding docs that do not omitTermFreqAndPositions for
// field X, then adding docs that do omitTermFreqAndPositions for that same
// field,
@ -230,21 +133,7 @@ public class TestOmitTf extends LuceneTestCase {
assertNoPrx(ram);
// now add some documents with positions, and check
// there is no prox after full merge
d = new Document();
f1 = newTextField("f1", "This field has positions", Field.Store.NO);
d.add(f1);
for(int i=0;i<30;i++)
writer.addDocument(d);
// force merge
writer.forceMerge(1);
// flush
writer.close();
assertNoPrx(ram);
ram.close();
}

8
lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
View File

@ -512,7 +512,7 @@ public class TestTermVectorsWriter extends LuceneTestCase {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
Document document = new Document();
FieldType customType2 = new FieldType(StringField.TYPE_NOT_STORED);
FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED);
customType2.setStoreTermVectors(true);
customType2.setStoreTermVectorPositions(true);
customType2.setStoreTermVectorOffsets(true);
@ -524,7 +524,7 @@ public class TestTermVectorsWriter extends LuceneTestCase {
// Make first segment
iw.commit();
FieldType customType = new FieldType(StringField.TYPE_NOT_STORED);
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.setStoreTermVectors(true);
document = new Document();
document.add(newField("tvtest", "a b c", customType));
@ -542,7 +542,7 @@ public class TestTermVectorsWriter extends LuceneTestCase {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
Document document = new Document();
FieldType customType = new FieldType(StringField.TYPE_NOT_STORED);
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.setStoreTermVectors(true);
document.add(newField("tvtest", "a b c", customType));
iw.addDocument(document);
@ -556,7 +556,7 @@ public class TestTermVectorsWriter extends LuceneTestCase {
iw.forceMerge(1);
FieldType customType2 = new FieldType(StringField.TYPE_NOT_STORED);
FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED);
customType2.setStoreTermVectors(true);
document.add(newField("tvtest", "a b c", customType2));
document = new Document();

2
lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
View File

@ -737,7 +737,7 @@ public class TestTermsEnum extends LuceneTestCase {
w.addDocument(doc);
doc = new Document();
doc.add(newStringField("field", "bbb", Field.Store.NO));
doc.add(newTextField("field", "bbb", Field.Store.NO));
w.addDocument(doc);
doc = new Document();

2
solr/core/src/test-files/solr/configsets/backcompat/conf/schema.xml
View File

@ -17,6 +17,6 @@
-->
<schema name="minimal" version="1.1">
<fieldType name="string" class="solr.StrField"/>
<field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
<field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" omitTermFreqAndPositions="true" />
<uniquekey>id</uniquekey>
</schema>