[Upgrade] Lucene-9.2.0-snapshot-ba8c3a8 (#3416)

Upgrades to latest snapshot of lucene 9.2.0 in preparation for GA release.

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>

buildSrc/src/main/resources/forbidden/opensearch-test-signatures.txt

@@ -19,7 +19,6 @@ com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded see
 com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded repeats
 
 org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
-org.apache.lucene.tests.util.LuceneTestCase$Slow @ Don't write slow tests
 org.junit.Ignore @ Use AwaitsFix instead
 org.apache.lucene.tests.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
 com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.2.0-snapshot-f4f1f70
+lucene            = 9.2.0-snapshot-ba8c3a8
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.3+7

libs/core/src/test/java/org/opensearch/core/internal/io/IOUtilsTests.java

@@ -40,7 +40,6 @@ import org.opensearch.test.OpenSearchTestCase;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.net.URI;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
@@ -172,10 +171,8 @@ public class IOUtilsTests extends OpenSearchTestCase {
         for (int i = 0; i < numberOfLocations; i++) {
             if (exception && randomBoolean()) {
                 final Path location = createTempDir();
-                final FileSystem fs = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem()).getFileSystem(
+                final FilterFileSystemProvider ffsp = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem());
-                    URI.create("file:///")
+                final Path wrapped = ffsp.wrapPath(location);
-                );
-                final Path wrapped = new FilterPath(location, fs);
                 locations[i] = wrapped.resolve(randomAlphaOfLength(8));
                 Files.createDirectory(locations[i]);
                 locationsThrowingException.add(locations[i]);
@@ -256,8 +253,8 @@ public class IOUtilsTests extends OpenSearchTestCase {
 
     public void testFsyncAccessDeniedOpeningDirectory() throws Exception {
         final Path path = createTempDir().toRealPath();
-        final FileSystem fs = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem()).getFileSystem(URI.create("file:///"));
+        final FilterFileSystemProvider ffsp = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem());
-        final Path wrapped = new FilterPath(path, fs);
+        final Path wrapped = ffsp.wrapPath(path);
         if (Constants.WINDOWS) {
             // no exception, we early return and do not even try to open the directory
             IOUtils.fsync(wrapped, true);

modules/lang-expression/licenses/lucene-expressions-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+fe5e4cf94d26bbe1d982808f34fa132bba5565e3

modules/lang-expression/licenses/lucene-expressions-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-f2a8008e74589f77f1d3da305cf58c88ee01d1c1

plugins/analysis-icu/licenses/lucene-analysis-icu-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+9fc73c790c037e817635fcc30ea0891e6acd2fac

plugins/analysis-icu/licenses/lucene-analysis-icu-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-f9569365e80897f1a9161254d5d2f44a44f95db8

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+058ffd84388f9ffcf0bfdd7f43a6e832836a2927

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-35c14b39ff522dd6665e74a873035827b865075e

plugins/analysis-nori/licenses/lucene-analysis-nori-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+fb166f35ba04a7687b3073afb9972f6669ac722e

plugins/analysis-nori/licenses/lucene-analysis-nori-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-a84218a1ea0d5c52d6591d417061518b8a8be4e4

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+abf9eb24601ec11ce5b61e4753b41444a869b29d

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-057bbd20b15899844b23d2cf034a167b4fe581f0

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+b310130fe9e0f31ce4218cda921309b1143f3541

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-6a9a8d49e87b6999560a131e16234e46f21e6b42

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+ad8783255cdcb6e7ab23a505123716ad979f3484

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-fcad3608779e0b3ab8703903b9d28cdc32767d60

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+75f8fbb94a303d04c5dc2b25436300a463003dd6

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-bd28479bdf3c076bc89a0d30864188d444410267

server/licenses/lucene-analysis-common-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+b2aa0739c95f1f715f407087dbcf96c5c21f4cc7

server/licenses/lucene-analysis-common-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-1f7c4b91c8ef9f65e85c5190080b3f796076f355

server/licenses/lucene-backward-codecs-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+e4a2f89c03e98e0fc211bba2c090047a007eb442

server/licenses/lucene-backward-codecs-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-e3ee195405dc0cb249fe2eb3f3a6a848c4686645

server/licenses/lucene-core-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+06df30c240dfb970002c104d44370ae58b7cb60a

server/licenses/lucene-core-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-f1cb45d20f7f23c420c56a94e9153e96bfdd6e1f

server/licenses/lucene-grouping-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+1991e0f0f71c3c99ba726fcfa372f7ba7c75bcf0

server/licenses/lucene-grouping-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-29052ac8f5255c8df2bb1d3d0da94e112c181679

server/licenses/lucene-highlighter-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+cac793b5cfbccf5c310d51bc78cf97ce3befceac

server/licenses/lucene-highlighter-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-d710569c00d561c70d8290de4c4c15fe9735f94f

server/licenses/lucene-join-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+79344146c032fda532def9771de589c4798117e5

server/licenses/lucene-join-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-0f3081b32664d8ca6318e69dd054befb5f9a334b

server/licenses/lucene-memory-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+1d5f1c88786bcdfc50466f963ef07cbd9c6c7827

server/licenses/lucene-memory-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-81b63e23b87c054c140ff6a1e2e6696ca750d51c

server/licenses/lucene-misc-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+adba90f90cf6815eeb9009c1a42d7c86f916d9da

server/licenses/lucene-misc-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-0cdea200c1890b877d26ce58b7d797f122bb8328

server/licenses/lucene-queries-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+d0f38091eee45a118173c9201677ebafa9ed9e89

server/licenses/lucene-queries-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-50d8395e3afc502f267cb308399ab783edfabec0

server/licenses/lucene-queryparser-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+05e2ca9fc81e8b73f746c5ec40321d6d90e3bcdd

server/licenses/lucene-queryparser-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-815b394c8be5cbb9673011953da38d39a843b0fa

server/licenses/lucene-sandbox-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+c4ebbf7fd05e2889624b4dd9afb3f7b22aad94f3

server/licenses/lucene-sandbox-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-7bb7c539172dc3513d4f34e7f29d2cd3a0352361

server/licenses/lucene-spatial-extras-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+08836d9dee5a2e9e92b538023285de3d620abd4b

server/licenses/lucene-spatial-extras-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-7bc2f2e37f866e3b376f083e4b7cc89a8cb45fd0

server/licenses/lucene-spatial3d-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+d492d0c7b4bb76c3de7cfc1b4fe224ef9e9e7056

server/licenses/lucene-spatial3d-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-a5f79bb1f8337dbe6fc50fc5abd46d4eaaf4d433

server/licenses/lucene-suggest-9.2.0-snapshot-ba8c3a8.jar.sha1

@@ -0,0 +1 @@
+71b5b0cfb5b5809c4a86e947b1f4d9202d6f1b75

server/licenses/lucene-suggest-9.2.0-snapshot-f4f1f70.jar.sha1

@@ -1 +0,0 @@
-9e74f66171ea215e8c4936588381c0950c290c80

server/src/main/java/org/opensearch/common/lucene/Lucene.java

@@ -125,7 +125,7 @@ import java.util.Map;
  * @opensearch.internal
  */
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene91";
+    public static final String LATEST_CODEC = "Lucene92";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 

server/src/main/java/org/opensearch/index/codec/CodecService.java

@@ -34,8 +34,8 @@ package org.opensearch.index.codec;
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec.Mode;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.mapper.MapperService;
@@ -62,8 +62,8 @@ public class CodecService {
     public CodecService(@Nullable MapperService mapperService, Logger logger) {
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene91Codec());
+            codecs.put(DEFAULT_CODEC, new Lucene92Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene91Codec(Mode.BEST_COMPRESSION));
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Mode.BEST_COMPRESSION));
         } else {
             codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
             codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));

server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java

@@ -36,7 +36,7 @@ import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.index.mapper.CompletionFieldMapper;
@@ -53,7 +53,7 @@ import org.opensearch.index.mapper.MapperService;
  *
  * @opensearch.internal
  */
-public class PerFieldMappingPostingFormatCodec extends Lucene91Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene92Codec {
     private final Logger logger;
     private final MapperService mapperService;
     private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();

server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalMapping.java

@@ -112,4 +112,8 @@ final class GlobalOrdinalMapping extends SortedSetDocValues {
         return values.cost();
     }
 
+    @Override
+    public long docValueCount() {
+        return values.docValueCount();
+    }
 }

server/src/main/java/org/opensearch/index/fielddata/ordinals/MultiOrdinals.java

@@ -229,5 +229,10 @@ public class MultiOrdinals extends Ordinals {
         public BytesRef lookupOrd(long ord) {
             return values.lookupOrd(ord);
         }
+
+        @Override
+        public long docValueCount() {
+            return currentEndOffset - currentOffset;
+        }
     }
 }

server/src/main/java/org/opensearch/search/aggregations/support/MissingValues.java

@@ -310,6 +310,11 @@ public enum MissingValues {
                 return true;
             }
 
+            @Override
+            public long docValueCount() {
+                return values.docValueCount();
+            }
+
             @Override
             public String toString() {
                 return "anon AbstractSortedDocValues of [" + super.toString() + "]";
@@ -340,6 +345,11 @@ public enum MissingValues {
                 return 1 + values.getValueCount();
             }
 
+            @Override
+            public long docValueCount() {
+                return values.docValueCount();
+            }
+
             @Override
             public long nextOrd() throws IOException {
                 if (hasOrds) {

server/src/test/java/org/opensearch/index/codec/CodecTests.java

@@ -34,7 +34,7 @@ package org.opensearch.index.codec;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
@@ -65,21 +65,21 @@ public class CodecTests extends OpenSearchTestCase {
     public void testResolveDefaultCodecs() throws Exception {
         CodecService codecService = createCodecService();
         assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class));
-        assertThat(codecService.codec("default"), instanceOf(Lucene91Codec.class));
+        assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class));
     }
 
     public void testDefault() throws Exception {
         Codec codec = createCodecService().codec("default");
-        assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_SPEED, codec);
+        assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec);
     }
 
     public void testBestCompression() throws Exception {
         Codec codec = createCodecService().codec("best_compression");
-        assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_COMPRESSION, codec);
+        assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec);
     }
 
     // write some docs with it, inspect .si to see this was the used compression
-    private void assertStoredFieldsCompressionEquals(Lucene91Codec.Mode expected, Codec actual) throws Exception {
+    private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(null);
         iwc.setCodec(actual);
@@ -91,7 +91,7 @@ public class CodecTests extends OpenSearchTestCase {
         SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
         String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
         assertNotNull(v);
-        assertEquals(expected, Lucene91Codec.Mode.valueOf(v));
+        assertEquals(expected, Lucene92Codec.Mode.valueOf(v));
         ir.close();
         dir.close();
     }

server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java

@@ -32,7 +32,7 @@
 package org.opensearch.index.engine;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -70,7 +70,7 @@ public class CompletionStatsCacheTests extends OpenSearchTestCase {
     public void testCompletionStatsCache() throws IOException, InterruptedException {
         final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
         final PostingsFormat postingsFormat = new Completion90PostingsFormat();
-        indexWriterConfig.setCodec(new Lucene91Codec() {
+        indexWriterConfig.setCodec(new Lucene92Codec() {
             @Override
             public PostingsFormat getPostingsFormatForField(String field) {
                 return postingsFormat; // all fields are suggest fields

server/src/test/java/org/opensearch/search/MultiValueModeTests.java

@@ -763,6 +763,11 @@ public class MultiValueModeTests extends OpenSearchTestCase {
             public long getValueCount() {
                 return 1 << 20;
             }
+
+            @Override
+            public long docValueCount() {
+                return array[doc].length;
+            }
         };
         verifySortedSet(multiValues, numDocs);
         final FixedBitSet rootDocs = randomRootDocs(numDocs);

server/src/test/java/org/opensearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java

@@ -83,6 +83,10 @@ public class BinaryRangeAggregatorTests extends OpenSearchTestCase {
             return terms.length;
         }
 
+        @Override
+        public long docValueCount() {
+            return ords.length;
+        }
     }
 
     private void doTestSortedSetRangeLeafCollector(int maxNumValuesPerDoc) throws Exception {

server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java

@@ -98,6 +98,10 @@ public class IncludeExcludeTests extends OpenSearchTestCase {
                 return 1;
             }
 
+            @Override
+            public long docValueCount() {
+                return 1;
+            }
         };
         IncludeExclude inexcl = new IncludeExclude(new TreeSet<>(Collections.singleton(new BytesRef("foo"))), null);
         OrdinalsFilter filter = inexcl.convertToOrdinalsFilter(DocValueFormat.RAW);

server/src/test/java/org/opensearch/search/aggregations/support/MissingValuesTests.java

@@ -155,6 +155,11 @@ public class MissingValuesTests extends OpenSearchTestCase {
                     return NO_MORE_ORDS;
                 }
             }
+
+            @Override
+            public long docValueCount() {
+                return ords[doc].length;
+            }
         };
 
         final BytesRef existingMissing = RandomPicks.randomFrom(random(), values);
@@ -257,6 +262,11 @@ public class MissingValuesTests extends OpenSearchTestCase {
                 return values[Math.toIntExact(ord)];
             }
 
+            @Override
+            public long docValueCount() {
+                throw new UnsupportedOperationException();
+            }
+
             @Override
             public long getValueCount() {
                 return values.length;

test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java

@@ -221,6 +221,7 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         .put("spanishpluralstem", Void.class)
         // LUCENE-10352
         .put("daitchmokotoffsoundex", Void.class)
+        .put("persianstem", Void.class)
         .immutableMap();
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String, Class<?>>()