LUCENE-8316: Allow DV updates for not existing fields

Today we prevent DV updates for non-existing fields except
of the soft deletes case. Yet, this can cause inconsitent field numbers
etc. since we don't go through the global field number map etc. This
change removes the limitation of updating DVs in docs even if the field
doesn't exists. This also has the benefit that the error messages if
the field type doesn't match is consistent with what DWPT throws.

lucene/core/src/java/org/apache/lucene/index/FieldInfos.java

@@ -346,7 +346,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
     }
     
     synchronized Set<String> getFieldNames() {
-      return Collections.unmodifiableSet(new HashSet<String>(nameToNumber.keySet()));
+      return Collections.unmodifiableSet(new HashSet<>(nameToNumber.keySet()));
     }
 
     synchronized void clear() {

lucene/core/src/java/org/apache/lucene/index/IndexWriter.java

@@ -1775,8 +1775,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
       if (dvType == DocValuesType.NONE) {
         throw new IllegalArgumentException("can only update NUMERIC or BINARY fields! field=" + f.name());
       }
-      if (!globalFieldNumberMap.contains(f.name(), dvType) && f.name().equals(config.softDeletesField) == false) {
-        throw new IllegalArgumentException("can only update existing docvalues fields! field=" + f.name() + ", type=" + dvType);
+      if (globalFieldNumberMap.contains(f.name(), dvType) == false) {
+        // if this field doesn't exists we try to add it. if it exists and the DV type doesn't match we
+        // get a consistent error message as if you try to do that during an indexing operation.
+        globalFieldNumberMap.addOrGet(f.name(), -1, IndexOptions.NONE, dvType, 0, 0);
+        assert globalFieldNumberMap.contains(f.name(), dvType);
       }
       if (config.getIndexSortFields().contains(f.name())) {
         throw new IllegalArgumentException("cannot update docvalues field involved in the index sort, field=" + f.name() + ", sort=" + config.getIndexSort());
@@ -1807,11 +1810,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
     return docWriter.getNumDocs();
   }
 
-  // for test purpose
-  final synchronized Collection<String> getIndexFileNames() throws IOException {
-    return segmentInfos.files(true);
-  }
-
   // for test purpose
   final synchronized int maxDoc(int i) {
     if (i >= 0 && i < segmentInfos.size()) {

lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java

@@ -639,6 +639,56 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
     }
     IOUtils.close(writer, dir);
   }
-  
+
+  public void testUpdateNotExistingFieldDV() throws IOException {
+    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+    try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, conf)) {
+      Document doc = new Document();
+      doc.add(new StringField("id", "1", Store.YES));
+      doc.add(new NumericDocValuesField("test", 1));
+      writer.addDocument(doc);
+      if (random().nextBoolean()) {
+        writer.commit();
+      }
+      writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("not_existing", 1));
+
+      Document doc1 = new Document();
+      doc1.add(new StringField("id", "2", Store.YES));
+      doc1.add(new BinaryDocValuesField("not_existing", new BytesRef()));
+      IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+          writer.addDocument(doc1)
+      );
+      assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());
+
+      iae = expectThrows(IllegalArgumentException.class, () ->
+          writer.updateDocValues(new Term("id", "1"), new BinaryDocValuesField("not_existing", new BytesRef()))
+      );
+      assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());
+    }
+  }
+
+  public void testUpdateFieldWithNoPreviousDocValues() throws IOException {
+    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+    try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, conf)) {
+      Document doc = new Document();
+      doc.add(new StringField("id", "1", Store.YES));
+      writer.addDocument(doc);
+      if (random().nextBoolean()) {
+        try (DirectoryReader reader = writer.getReader()) {
+          NumericDocValues id = reader.leaves().get(0).reader().getNumericDocValues("id");
+          assertNull(id);
+        }
+      } else if (random().nextBoolean()) {
+        writer.commit();
+      }
+      writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("id", 1));
+      try (DirectoryReader reader = writer.getReader()) {
+        NumericDocValues id = reader.leaves().get(0).reader().getNumericDocValues("id");
+        assertNotNull(id);
+        assertTrue(id.advanceExact(0));
+        assertEquals(1, id.longValue());
+      }
+    }
+  }
 }