LUCENE-8595: Fix interleaved DV update and reset

This change fixes a bug where interleaved update and reset value
to the same doc in the same updates package looses an update
if the reset comes before the update as well as loosing the reset
if the update comes frist.

Co-authored-by: Adrien Grand <jpountz@gmail.com>

lucene/CHANGES.txt

@@ -312,6 +312,10 @@ Bug fixes
 * LUCENE-8586: Intervals.or() could get stuck in an infinite loop on certain indexes
   (Alan Woodward)
 
+* LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
+  to the same doc in the same updates package looses an update if the reset comes before
+  the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
+
 New Features
 
 * LUCENE-8496: Selective indexing - modify BKDReader/BKDWriter to allow users

lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java

@@ -298,7 +298,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
         // increasing docID order:
         // NOTE: we can have ties here, when the same docID was updated in the same segment, in which case we rely on sort being
         // stable and preserving original order so the last update to that docID wins
-        return Long.compare(docs.get(i), docs.get(j));
+        return Long.compare(docs.get(i)>>>1, docs.get(j)>>>1);
       }
     }.sort(0, size);
   }
@@ -392,9 +392,13 @@ abstract class DocValuesFieldUpdates implements Accountable {
       }
       long longDoc = docs.get(idx);
       ++idx;
-      while (idx < size && docs.get(idx) == longDoc) {
+      for (; idx < size; idx++) {
         // scan forward to last update to this doc
-        ++idx;
+        final long nextLongDoc = docs.get(idx);
+        if ((longDoc >>> 1) != (nextLongDoc >>> 1)) {
+          break;
+        }
+        longDoc = nextLongDoc;
       }
       hasValue = (longDoc & HAS_VALUE_MASK) >  0;
       if (hasValue) {

lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java

@@ -69,4 +69,67 @@ public class TestDocValuesFieldUpdates extends LuceneTestCase {
     assertEquals(24, iterator.longValue());
     assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
   }
+
+  public void testUpdateAndResetSameDoc() {
+    NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 2);
+    updates.add(0, 1);
+    updates.reset(0);
+    updates.finish();
+    NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+    assertEquals(0, iterator.nextDoc());
+    assertFalse(iterator.hasValue());
+    assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
+  }
+
+  public void testUpdateAndResetUpdateSameDoc() {
+    NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 3);
+    updates.add(0, 1);
+    updates.add(0);
+    updates.add(0, 2);
+    updates.finish();
+    NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+    assertEquals(0, iterator.nextDoc());
+    assertTrue(iterator.hasValue());
+    assertEquals(2, iterator.longValue());
+    assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
+  }
+
+  public void testUpdatesAndResetRandom() {
+    NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 10);
+    int numUpdates = 10 + random().nextInt(100);
+    Integer[] values = new Integer[5];
+    for (int i = 0; i < 5; i++) {
+      values[i] = random().nextBoolean() ? null : random().nextInt(100);
+      if (values[i] == null) {
+        updates.reset(i);
+      } else {
+        updates.add(i, values[i]);
+      }
+    }
+    for (int i = 0; i < numUpdates; i++) {
+      int docId = random().nextInt(5);
+      values[docId] = random().nextBoolean() ? null : random().nextInt(100);
+      if (values[docId] == null) {
+        updates.reset(docId);
+      } else {
+        updates.add(docId, values[docId]);
+      }
+    }
+
+    updates.finish();
+    NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+    int idx = 0;
+    while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+      assertEquals(idx, iterator.docID());
+      if (values[idx] == null) {
+        assertFalse(iterator.hasValue());
+      } else {
+        assertTrue(iterator.hasValue());
+        assertEquals(values[idx].longValue(), iterator.longValue());
+      }
+      idx++;
+    }
+
+
+  }
 }