honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Upgrade] Lucene-9.2.0-snapshot-ba8c3a8 (#3416) 

Upgrades to latest snapshot of lucene 9.2.0 in preparation for GA release.

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
This commit is contained in:
Nick Knize 2022-05-20 16:29:51 -05:00 committed by GitHub
parent acf7da78f9
commit 5358502107
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
60 changed files with 85 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
buildSrc/src/main/resources/forbidden/opensearch-test-signatures.txt
View File

@ -19,7 +19,6 @@ com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded see
com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded repeats
org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
org.apache.lucene.tests.util.LuceneTestCase$Slow @ Don't write slow tests
org.junit.Ignore @ Use AwaitsFix instead
org.apache.lucene.tests.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!

2
buildSrc/version.properties
View File

@ -1,5 +1,5 @@
opensearch = 3.0.0
lucene = 9.2.0-snapshot-f4f1f70
lucene = 9.2.0-snapshot-ba8c3a8
bundled_jdk_vendor = adoptium
bundled_jdk = 17.0.3+7

11
libs/core/src/test/java/org/opensearch/core/internal/io/IOUtilsTests.java
View File

@ -40,7 +40,6 @@ import org.opensearch.test.OpenSearchTestCase;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
@ -172,10 +171,8 @@ public class IOUtilsTests extends OpenSearchTestCase {
for (int i = 0; i < numberOfLocations; i++) {
if (exception && randomBoolean()) {
final Path location = createTempDir();
final FileSystem fs = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem()).getFileSystem(
URI.create("file:///")
);
final Path wrapped = new FilterPath(location, fs);
final FilterFileSystemProvider ffsp = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem());
final Path wrapped = ffsp.wrapPath(location);
locations[i] = wrapped.resolve(randomAlphaOfLength(8));
Files.createDirectory(locations[i]);
locationsThrowingException.add(locations[i]);
@ -256,8 +253,8 @@ public class IOUtilsTests extends OpenSearchTestCase {
public void testFsyncAccessDeniedOpeningDirectory() throws Exception {
final Path path = createTempDir().toRealPath();
final FileSystem fs = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem()).getFileSystem(URI.create("file:///"));
final Path wrapped = new FilterPath(path, fs);
final FilterFileSystemProvider ffsp = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem());
final Path wrapped = ffsp.wrapPath(path);
if (Constants.WINDOWS) {
// no exception, we early return and do not even try to open the directory
IOUtils.fsync(wrapped, true);

1
modules/lang-expression/licenses/lucene-expressions-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
fe5e4cf94d26bbe1d982808f34fa132bba5565e3

1
modules/lang-expression/licenses/lucene-expressions-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
f2a8008e74589f77f1d3da305cf58c88ee01d1c1

1
plugins/analysis-icu/licenses/lucene-analysis-icu-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
9fc73c790c037e817635fcc30ea0891e6acd2fac

1
plugins/analysis-icu/licenses/lucene-analysis-icu-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
f9569365e80897f1a9161254d5d2f44a44f95db8

1
plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
058ffd84388f9ffcf0bfdd7f43a6e832836a2927

1
plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
35c14b39ff522dd6665e74a873035827b865075e

1
plugins/analysis-nori/licenses/lucene-analysis-nori-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
fb166f35ba04a7687b3073afb9972f6669ac722e

1
plugins/analysis-nori/licenses/lucene-analysis-nori-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
a84218a1ea0d5c52d6591d417061518b8a8be4e4

1
plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
abf9eb24601ec11ce5b61e4753b41444a869b29d

1
plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
057bbd20b15899844b23d2cf034a167b4fe581f0

1
plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
b310130fe9e0f31ce4218cda921309b1143f3541

1
plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
6a9a8d49e87b6999560a131e16234e46f21e6b42

1
plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ad8783255cdcb6e7ab23a505123716ad979f3484

1
plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
fcad3608779e0b3ab8703903b9d28cdc32767d60

1
plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
75f8fbb94a303d04c5dc2b25436300a463003dd6

1
plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
bd28479bdf3c076bc89a0d30864188d444410267

1
server/licenses/lucene-analysis-common-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
b2aa0739c95f1f715f407087dbcf96c5c21f4cc7

1
server/licenses/lucene-analysis-common-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
1f7c4b91c8ef9f65e85c5190080b3f796076f355

1
server/licenses/lucene-backward-codecs-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e4a2f89c03e98e0fc211bba2c090047a007eb442

1
server/licenses/lucene-backward-codecs-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
e3ee195405dc0cb249fe2eb3f3a6a848c4686645

1
server/licenses/lucene-core-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
06df30c240dfb970002c104d44370ae58b7cb60a

1
server/licenses/lucene-core-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
f1cb45d20f7f23c420c56a94e9153e96bfdd6e1f

1
server/licenses/lucene-grouping-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
1991e0f0f71c3c99ba726fcfa372f7ba7c75bcf0

1
server/licenses/lucene-grouping-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
29052ac8f5255c8df2bb1d3d0da94e112c181679

1
server/licenses/lucene-highlighter-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
cac793b5cfbccf5c310d51bc78cf97ce3befceac

1
server/licenses/lucene-highlighter-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
d710569c00d561c70d8290de4c4c15fe9735f94f

1
server/licenses/lucene-join-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
79344146c032fda532def9771de589c4798117e5

1
server/licenses/lucene-join-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
0f3081b32664d8ca6318e69dd054befb5f9a334b

1
server/licenses/lucene-memory-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
1d5f1c88786bcdfc50466f963ef07cbd9c6c7827

1
server/licenses/lucene-memory-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
81b63e23b87c054c140ff6a1e2e6696ca750d51c

1
server/licenses/lucene-misc-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
adba90f90cf6815eeb9009c1a42d7c86f916d9da

1
server/licenses/lucene-misc-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
0cdea200c1890b877d26ce58b7d797f122bb8328

1
server/licenses/lucene-queries-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d0f38091eee45a118173c9201677ebafa9ed9e89

1
server/licenses/lucene-queries-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
50d8395e3afc502f267cb308399ab783edfabec0

1
server/licenses/lucene-queryparser-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
05e2ca9fc81e8b73f746c5ec40321d6d90e3bcdd

1
server/licenses/lucene-queryparser-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
815b394c8be5cbb9673011953da38d39a843b0fa

1
server/licenses/lucene-sandbox-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
c4ebbf7fd05e2889624b4dd9afb3f7b22aad94f3

1
server/licenses/lucene-sandbox-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
7bb7c539172dc3513d4f34e7f29d2cd3a0352361

1
server/licenses/lucene-spatial-extras-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
08836d9dee5a2e9e92b538023285de3d620abd4b

1
server/licenses/lucene-spatial-extras-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
7bc2f2e37f866e3b376f083e4b7cc89a8cb45fd0

1
server/licenses/lucene-spatial3d-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d492d0c7b4bb76c3de7cfc1b4fe224ef9e9e7056

1
server/licenses/lucene-spatial3d-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
a5f79bb1f8337dbe6fc50fc5abd46d4eaaf4d433

1
server/licenses/lucene-suggest-9.2.0-snapshot-ba8c3a8.jar.sha1 Normal file
View File

@ -0,0 +1 @@
71b5b0cfb5b5809c4a86e947b1f4d9202d6f1b75

1
server/licenses/lucene-suggest-9.2.0-snapshot-f4f1f70.jar.sha1
View File

@ -1 +0,0 @@
9e74f66171ea215e8c4936588381c0950c290c80

2
server/src/main/java/org/opensearch/common/lucene/Lucene.java
View File

@ -125,7 +125,7 @@ import java.util.Map;
* @opensearch.internal
*/
public class Lucene {
public static final String LATEST_CODEC = "Lucene91";
public static final String LATEST_CODEC = "Lucene92";
public static final String SOFT_DELETES_FIELD = "__soft_deletes";

8
server/src/main/java/org/opensearch/index/codec/CodecService.java
View File

@ -34,8 +34,8 @@ package org.opensearch.index.codec;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene91.Lucene91Codec.Mode;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.index.mapper.MapperService;
@ -62,8 +62,8 @@ public class CodecService {
public CodecService(@Nullable MapperService mapperService, Logger logger) {
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene91Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene91Codec(Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene92Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Mode.BEST_COMPRESSION));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));

4
server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java
View File

@ -36,7 +36,7 @@ import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.index.mapper.CompletionFieldMapper;
@ -53,7 +53,7 @@ import org.opensearch.index.mapper.MapperService;
*
* @opensearch.internal
*/
public class PerFieldMappingPostingFormatCodec extends Lucene91Codec {
public class PerFieldMappingPostingFormatCodec extends Lucene92Codec {
private final Logger logger;
private final MapperService mapperService;
private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();

4
server/src/main/java/org/opensearch/index/fielddata/ordinals/GlobalOrdinalMapping.java
View File

@ -112,4 +112,8 @@ final class GlobalOrdinalMapping extends SortedSetDocValues {
return values.cost();
}
@Override
public long docValueCount() {
return values.docValueCount();
}
}

5
server/src/main/java/org/opensearch/index/fielddata/ordinals/MultiOrdinals.java
View File

@ -229,5 +229,10 @@ public class MultiOrdinals extends Ordinals {
public BytesRef lookupOrd(long ord) {
return values.lookupOrd(ord);
}
@Override
public long docValueCount() {
return currentEndOffset - currentOffset;
}
}
}

10
server/src/main/java/org/opensearch/search/aggregations/support/MissingValues.java
View File

@ -310,6 +310,11 @@ public enum MissingValues {
return true;
}
@Override
public long docValueCount() {
return values.docValueCount();
}
@Override
public String toString() {
return "anon AbstractSortedDocValues of [" + super.toString() + "]";
@ -340,6 +345,11 @@ public enum MissingValues {
return 1 + values.getValueCount();
}
@Override
public long docValueCount() {
return values.docValueCount();
}
@Override
public long nextOrd() throws IOException {
if (hasOrds) {

12
server/src/test/java/org/opensearch/index/codec/CodecTests.java
View File

@ -34,7 +34,7 @@ package org.opensearch.index.codec;
import org.apache.logging.log4j.LogManager;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
@ -65,21 +65,21 @@ public class CodecTests extends OpenSearchTestCase {
public void testResolveDefaultCodecs() throws Exception {
CodecService codecService = createCodecService();
assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene91Codec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class));
}
public void testDefault() throws Exception {
Codec codec = createCodecService().codec("default");
assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_SPEED, codec);
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec);
}
public void testBestCompression() throws Exception {
Codec codec = createCodecService().codec("best_compression");
assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_COMPRESSION, codec);
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec);
}
// write some docs with it, inspect .si to see this was the used compression
private void assertStoredFieldsCompressionEquals(Lucene91Codec.Mode expected, Codec actual) throws Exception {
private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(null);
iwc.setCodec(actual);
@ -91,7 +91,7 @@ public class CodecTests extends OpenSearchTestCase {
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
assertNotNull(v);
assertEquals(expected, Lucene91Codec.Mode.valueOf(v));
assertEquals(expected, Lucene92Codec.Mode.valueOf(v));
ir.close();
dir.close();
}

4
server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java
View File

@ -32,7 +32,7 @@
package org.opensearch.index.engine;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
@ -70,7 +70,7 @@ public class CompletionStatsCacheTests extends OpenSearchTestCase {
public void testCompletionStatsCache() throws IOException, InterruptedException {
final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
final PostingsFormat postingsFormat = new Completion90PostingsFormat();
indexWriterConfig.setCodec(new Lucene91Codec() {
indexWriterConfig.setCodec(new Lucene92Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return postingsFormat; // all fields are suggest fields

5
server/src/test/java/org/opensearch/search/MultiValueModeTests.java
View File

@ -763,6 +763,11 @@ public class MultiValueModeTests extends OpenSearchTestCase {
public long getValueCount() {
return 1 << 20;
}
@Override
public long docValueCount() {
return array[doc].length;
}
};
verifySortedSet(multiValues, numDocs);
final FixedBitSet rootDocs = randomRootDocs(numDocs);

4
server/src/test/java/org/opensearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
View File

@ -83,6 +83,10 @@ public class BinaryRangeAggregatorTests extends OpenSearchTestCase {
return terms.length;
}
@Override
public long docValueCount() {
return ords.length;
}
}
private void doTestSortedSetRangeLeafCollector(int maxNumValuesPerDoc) throws Exception {

4
server/src/test/java/org/opensearch/search/aggregations/support/IncludeExcludeTests.java
View File

@ -98,6 +98,10 @@ public class IncludeExcludeTests extends OpenSearchTestCase {
return 1;
}
@Override
public long docValueCount() {
return 1;
}
};
IncludeExclude inexcl = new IncludeExclude(new TreeSet<>(Collections.singleton(new BytesRef("foo"))), null);
OrdinalsFilter filter = inexcl.convertToOrdinalsFilter(DocValueFormat.RAW);

10
server/src/test/java/org/opensearch/search/aggregations/support/MissingValuesTests.java
View File

@ -155,6 +155,11 @@ public class MissingValuesTests extends OpenSearchTestCase {
return NO_MORE_ORDS;
}
}
@Override
public long docValueCount() {
return ords[doc].length;
}
};
final BytesRef existingMissing = RandomPicks.randomFrom(random(), values);
@ -257,6 +262,11 @@ public class MissingValuesTests extends OpenSearchTestCase {
return values[Math.toIntExact(ord)];
}
@Override
public long docValueCount() {
throw new UnsupportedOperationException();
}
@Override
public long getValueCount() {
return values.length;

1
test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
View File

@ -221,6 +221,7 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
.put("spanishpluralstem", Void.class)
// LUCENE-10352
.put("daitchmokotoffsoundex", Void.class)
.put("persianstem", Void.class)
.immutableMap();
static final Map<String, Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String, Class<?>>()