Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-7491: fix merge exception if the same field has points in some segments but not in others

This commit is contained in:
Mike McCandless 2016-10-12 09:00:26 -04:00
parent 6512d0c620
commit 1b7a88f61e
5 changed files with 40 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/CHANGES.txt
View File

@ -73,6 +73,11 @@ Bug Fixes
* LUCENE-7486: DisjunctionMaxQuery does not work correctly with queries that
return negative scores. (Ivan Provalov, Uwe Schindler, Adrien Grand)
* LUCENE-7491: Suddenly turning on dimensional points for some fields
that already exist in an index but didn't previously index
dimensional points could cause unexpected merge exceptions (Hans
Lund, Mike McCandless)
Improvements
* LUCENE-7439: FuzzyQuery now matches all terms within the specified

7
lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
View File

@ -47,7 +47,7 @@ public abstract class PointsWriter implements Closeable {
PointsReader pointsReader = mergeState.pointsReaders[i];
if (pointsReader != null) {
FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);
if (readerFieldInfo != null) {
if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
maxPointCount += pointsReader.size(fieldInfo.name);
docCount += pointsReader.getDocCount(fieldInfo.name);
}
@ -75,6 +75,11 @@ public abstract class PointsWriter implements Closeable {
continue;
}
if (readerFieldInfo.getPointDimensionCount() == 0) {
// This segment saw this field, but the field did not index points in it:
continue;
}
MergeState.DocMap docMap = mergeState.docMaps[i];
pointsReader.intersect(fieldInfo.name,
new IntersectVisitor() {

5
lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
View File

@ -165,7 +165,7 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
if (reader != null) {
FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
if (readerFieldInfo != null) {
if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
totMaxSize += reader.size(fieldInfo.name);
singleValuePerDoc &= reader.size(fieldInfo.name) == reader.getDocCount(fieldInfo.name);
}
@ -200,10 +200,9 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
// reader's FieldInfo as we do below) because field numbers can easily be different
// when addIndexes(Directory...) copies over segments from another index:
FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
if (readerFieldInfo != null) {
if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
BKDReader bkdReader = reader60.readers.get(readerFieldInfo.number);
if (bkdReader != null) {
bkdReaders.add(bkdReader);

4
lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
View File

@ -145,6 +145,8 @@ public final class FieldInfo {
if (this.pointDimensionCount == 0 && dimensionCount != 0) {
this.pointDimensionCount = dimensionCount;
this.pointNumBytes = dimensionNumBytes;
} else if (this.pointDimensionCount != dimensionCount || this.pointNumBytes != dimensionNumBytes) {
throw new IllegalArgumentException("cannot change field \"" + name + "\" from points dimensionCount=" + this.pointDimensionCount + ", numBytes=" + this.pointNumBytes + " to inconsistent dimensionCount=" + dimensionCount + ", numBytes=" + dimensionNumBytes);
}
if (this.indexOptions != IndexOptions.NONE) { // if updated field data is not for indexing, leave the updates out
@ -187,6 +189,8 @@ public final class FieldInfo {
pointDimensionCount = count;
pointNumBytes = numBytes;
assert checkConsistency();
}
/** Return point dimension count */

23
lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
View File

@ -997,4 +997,27 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
// structure than the tree created by adding points separately
return false;
}
// LUCENE-7491
public void testMixedSchema() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig();
RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
iwc.setMaxBufferedDocs(2);
for(int i=0;i<2;i++) {
Document doc = new Document();
doc.add(new StringField("id", Integer.toString(i), Field.Store.NO));
doc.add(new IntPoint("int", i));
w.addDocument(doc);
}
// index has 1 segment now (with 2 docs) and that segment does have points, but the "id" field in particular does NOT
Document doc = new Document();
doc.add(new IntPoint("id", 0));
w.addDocument(doc);
// now we write another segment where the id field does have points:
w.forceMerge(1);
IOUtils.close(w, dir);
}
}