LUCENE-7491: fix merge exception if the same field has points in some segments but not in others

lucene/CHANGES.txt

@@ -73,6 +73,11 @@ Bug Fixes
 * LUCENE-7486: DisjunctionMaxQuery does not work correctly with queries that
   return negative scores.  (Ivan Provalov, Uwe Schindler, Adrien Grand)
 
+* LUCENE-7491: Suddenly turning on dimensional points for some fields
+  that already exist in an index but didn't previously index
+  dimensional points could cause unexpected merge exceptions (Hans
+  Lund, Mike McCandless)
+
 Improvements
 
 * LUCENE-7439: FuzzyQuery now matches all terms within the specified

lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java

@@ -47,7 +47,7 @@ public abstract class PointsWriter implements Closeable {
       PointsReader pointsReader = mergeState.pointsReaders[i];
       if (pointsReader != null) {
         FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);
-        if (readerFieldInfo != null) {
+        if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
           maxPointCount += pointsReader.size(fieldInfo.name);
           docCount += pointsReader.getDocCount(fieldInfo.name);
         }
@@ -75,6 +75,11 @@ public abstract class PointsWriter implements Closeable {
                        continue;
                      }
 
+                     if (readerFieldInfo.getPointDimensionCount() == 0) {
+                       // This segment saw this field, but the field did not index points in it:
+                       continue;
+                     }
+
                      MergeState.DocMap docMap = mergeState.docMaps[i];
                      pointsReader.intersect(fieldInfo.name,
                                             new IntersectVisitor() {

lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java

@@ -165,7 +165,7 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
             if (reader != null) {
               FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
               FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
-              if (readerFieldInfo != null) {
+              if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
                 totMaxSize += reader.size(fieldInfo.name);
                 singleValuePerDoc &= reader.size(fieldInfo.name) == reader.getDocCount(fieldInfo.name);
               }
@@ -200,10 +200,9 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
                 // reader's FieldInfo as we do below) because field numbers can easily be different
                 // when addIndexes(Directory...) copies over segments from another index:
 
-
                 FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
                 FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
-                if (readerFieldInfo != null) {
+                if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
                   BKDReader bkdReader = reader60.readers.get(readerFieldInfo.number);
                   if (bkdReader != null) {
                     bkdReaders.add(bkdReader);

lucene/core/src/java/org/apache/lucene/index/FieldInfo.java

@@ -145,6 +145,8 @@ public final class FieldInfo {
     if (this.pointDimensionCount == 0 && dimensionCount != 0) {
       this.pointDimensionCount = dimensionCount;
       this.pointNumBytes = dimensionNumBytes;
+    } else if (this.pointDimensionCount != dimensionCount || this.pointNumBytes != dimensionNumBytes) {
+      throw new IllegalArgumentException("cannot change field \"" + name + "\" from points dimensionCount=" + this.pointDimensionCount + ", numBytes=" + this.pointNumBytes + " to inconsistent dimensionCount=" + dimensionCount + ", numBytes=" + dimensionNumBytes);
     }
 
     if (this.indexOptions != IndexOptions.NONE) { // if updated field data is not for indexing, leave the updates out
@@ -187,6 +189,8 @@ public final class FieldInfo {
 
     pointDimensionCount = count;
     pointNumBytes = numBytes;
+
+    assert checkConsistency();
   }
 
   /** Return point dimension count */

lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java

@@ -997,4 +997,27 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
     // structure than the tree created by adding points separately
     return false;
   }
+
+  // LUCENE-7491
+  public void testMixedSchema() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = newIndexWriterConfig();
+    RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+    iwc.setMaxBufferedDocs(2);
+    for(int i=0;i<2;i++) {
+      Document doc = new Document();
+      doc.add(new StringField("id", Integer.toString(i), Field.Store.NO));
+      doc.add(new IntPoint("int", i));
+      w.addDocument(doc);
+    }
+    // index has 1 segment now (with 2 docs) and that segment does have points, but the "id" field in particular does NOT
+
+    Document doc = new Document();
+    doc.add(new IntPoint("id", 0));
+    w.addDocument(doc);
+    // now we write another segment where the id field does have points:
+    
+    w.forceMerge(1);
+    IOUtils.close(w, dir);
+  }
 }