Update lucene version to 8.8.2 (#557)
This commit updates the codebase to the latest released version of Lucene. Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
This commit is contained in:
Nick Knize
GitHub
parent
821c44c440
commit
c5a3c3cb41
|
|
@ -1,5 +1,5 @@
|
|||
opensearch = 1.0.0
|
||||
lucene = 8.7.0
|
||||
lucene = 8.8.2
|
||||
|
||||
bundled_jdk_vendor = adoptopenjdk
|
||||
bundled_jdk = 15.0.1+9
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
71a442ef5ba4ff2543a69079b4a7f6f9d4f1f3d0
|
||||
|
|
@ -0,0 +1 @@
|
|||
85c06b0d5b966e6d8a75dd568a6eb1a7c3b34bdd
|
||||
|
|
@ -1 +0,0 @@
|
|||
38f7d348e534c8d37482fcba75f4546df8e143af
|
||||
|
|
@ -0,0 +1 @@
|
|||
dcda3ccd38ca68e23ec57c8ca16d13553066672b
|
||||
|
|
@ -1 +0,0 @@
|
|||
f301e64abaf39204a5b5eb2c55c5152c7328e938
|
||||
|
|
@ -0,0 +1 @@
|
|||
829b06a76ffaf0e551c5cda4a1e56990080ea6de
|
||||
|
|
@ -1 +0,0 @@
|
|||
3462eaa9b58e995c8e28060bf7ac2b054ca5e44e
|
||||
|
|
@ -0,0 +1 @@
|
|||
2c23e33c49fb255d96e0bf42ca7c58f030011727
|
||||
|
|
@ -1 +0,0 @@
|
|||
c5d438642d316cf2484d5a9f83ba8d35cdc28d5c
|
||||
|
|
@ -0,0 +1 @@
|
|||
b0ccf32c2baa88815b89c157f19b264a24958180
|
||||
|
|
@ -1 +0,0 @@
|
|||
6a58f7a9ac3562aecd2cfaf9a1097595e335be0e
|
||||
|
|
@ -0,0 +1 @@
|
|||
d7fb0c151dccc1954547acd3289cdf1369cda969
|
||||
|
|
@ -1 +0,0 @@
|
|||
22435907b6facd68e68300b329cc967137272861
|
||||
|
|
@ -0,0 +1 @@
|
|||
ca1b3379c99553c64d6b76271a5701f2bc2a2ce4
|
||||
|
|
@ -1 +0,0 @@
|
|||
a18c978ce5719b614445ca8f8075974df91d2631
|
||||
|
|
@ -0,0 +1 @@
|
|||
2070b7dd21682b4b9bd23f77146a73d0e3f3b05f
|
||||
|
|
@ -1 +0,0 @@
|
|||
fc219a023416a6fe6172f61d0d7bf64ed3645969
|
||||
|
|
@ -0,0 +1 @@
|
|||
fa4b3c1d847f4bb91a3afaa990889dd164883d86
|
||||
|
|
@ -1 +0,0 @@
|
|||
50c0493acdc9265d3a91365404e903bf408ab370
|
||||
|
|
@ -0,0 +1 @@
|
|||
f4a0475a1404d7be87e911f5c85f8b913ec9770b
|
||||
|
|
@ -1 +0,0 @@
|
|||
ed64084a1502c2a6a411cbd9826131b81e0bf07f
|
||||
|
|
@ -0,0 +1 @@
|
|||
e039fa90ee900c475bbc0943989ed5dccc556d3e
|
||||
|
|
@ -1 +0,0 @@
|
|||
90a094913f46093745e779cce4d5548d03a824c9
|
||||
|
|
@ -0,0 +1 @@
|
|||
207ac15287656b6ce61ea921fefec3070a8dd99d
|
||||
|
|
@ -1 +0,0 @@
|
|||
84c88d872e8680ff334b25c2d85336a26b1c776a
|
||||
|
|
@ -0,0 +1 @@
|
|||
2674644580108335c08a25fd2e0f0c7dee010700
|
||||
|
|
@ -1 +0,0 @@
|
|||
c2e04806a44227ff4c17530c6a0c8468d535b72e
|
||||
|
|
@ -0,0 +1 @@
|
|||
6eb956741169fe8a5643fa39b14aef2fc067d865
|
||||
|
|
@ -1 +0,0 @@
|
|||
52a346554a6160bfe1d32d93410d12bc2a795e0c
|
||||
|
|
@ -0,0 +1 @@
|
|||
63b6e8e862c898d263b941ffa55fe6c85bcdf2df
|
||||
|
|
@ -1 +0,0 @@
|
|||
e85fb21faa1ca9bd2c0f1283d9930adb058773e4
|
||||
|
|
@ -0,0 +1 @@
|
|||
a3a916c27c42ff6773c52d0be4045d4e5c00f7f8
|
||||
|
|
@ -1 +0,0 @@
|
|||
591f25f917a11143fbd9f4573468a0a8b5af21f2
|
||||
|
|
@ -0,0 +1 @@
|
|||
1f324c3a6e1a7634bbada993b2c014d12d09cb83
|
||||
|
|
@ -1 +0,0 @@
|
|||
66ae10cb549dea23346a4f5230de9f2f4880def4
|
||||
|
|
@ -0,0 +1 @@
|
|||
67c6580f8a5187fdb7f9eb2404bdecc4635b3328
|
||||
|
|
@ -1 +0,0 @@
|
|||
b17d87dc165ddb58d3e6f8a6fe1622cb67bcd5c5
|
||||
|
|
@ -0,0 +1 @@
|
|||
b674deb4957a64db5c8f4afb9127400f4e65c334
|
||||
|
|
@ -1 +0,0 @@
|
|||
cbdb38220c0f1974385eb928c5a80ee9b440015f
|
||||
|
|
@ -0,0 +1 @@
|
|||
227672b3b88c877b2df27608850ca15d834ec35c
|
||||
|
|
@ -1 +0,0 @@
|
|||
cd926effe54ef0eed0cfb6507cf71278f6f2c124
|
||||
|
|
@ -0,0 +1 @@
|
|||
eb77b665a3b3e9ca6b3bb82c3342b843d9729d0d
|
||||
|
|
@ -1 +0,0 @@
|
|||
07239e7962c99cfc976a21c4edb4b00c6d7db3b9
|
||||
|
|
@ -0,0 +1 @@
|
|||
36fd5301e7143f0fcb0ba933f96148dc1e132b66
|
||||
|
|
@ -70,7 +70,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
|
|||
public static final int V_EMPTY_ID = 0;
|
||||
public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST);
|
||||
|
||||
public static final Version V_1_0_0 = new Version(1000099, org.apache.lucene.util.Version.LUCENE_8_7_0);
|
||||
public static final Version V_1_0_0 = new Version(1000099, org.apache.lucene.util.Version.LUCENE_8_8_2);
|
||||
public static final Version CURRENT = V_1_0_0;
|
||||
|
||||
public static Version readVersion(StreamInput in) throws IOException {
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ package org.opensearch.index.codec;
|
|||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec.Mode;
|
||||
import org.opensearch.common.Nullable;
|
||||
import org.opensearch.common.collect.MapBuilder;
|
||||
import org.opensearch.index.mapper.MapperService;
|
||||
|
|
|
|||
|
|
@ -34,8 +34,9 @@ package org.opensearch.index.codec;
|
|||
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.DocValuesFormat;
|
||||
import org.apache.lucene.codecs.PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec;
|
||||
import org.opensearch.common.lucene.Lucene;
|
||||
import org.opensearch.index.mapper.CompletionFieldMapper;
|
||||
|
|
@ -53,13 +54,14 @@ import org.opensearch.index.mapper.MapperService;
|
|||
public class PerFieldMappingPostingFormatCodec extends Lucene87Codec {
|
||||
private final Logger logger;
|
||||
private final MapperService mapperService;
|
||||
private final DocValuesFormat dvFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION);
|
||||
|
||||
static {
|
||||
assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMappingPostingFormatCodec.class) :
|
||||
"PerFieldMappingPostingFormatCodec must subclass the latest " + "lucene codec: " + Lucene.LATEST_CODEC;
|
||||
}
|
||||
|
||||
public PerFieldMappingPostingFormatCodec(Lucene87StoredFieldsFormat.Mode compressionMode, MapperService mapperService, Logger logger) {
|
||||
public PerFieldMappingPostingFormatCodec(Mode compressionMode, MapperService mapperService, Logger logger) {
|
||||
super(compressionMode);
|
||||
this.mapperService = mapperService;
|
||||
this.logger = logger;
|
||||
|
|
@ -76,4 +78,8 @@ public class PerFieldMappingPostingFormatCodec extends Lucene87Codec {
|
|||
return super.getPostingsFormatForField(field);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DocValuesFormat getDocValuesFormatForField(String field) {
|
||||
return dvFormat;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@
|
|||
package org.opensearch.index.query;
|
||||
|
||||
import org.apache.lucene.document.LatLonShape;
|
||||
import org.apache.lucene.geo.GeoEncodingUtils;
|
||||
import org.apache.lucene.geo.LatLonGeometry;
|
||||
import org.apache.lucene.search.MatchNoDocsQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
|
|
@ -155,7 +156,12 @@ public class VectorGeoShapeQueryProcessor {
|
|||
@Override
|
||||
public Void visit(Point point) {
|
||||
if (point.isEmpty() == false) {
|
||||
geometries.add(GeoShapeUtils.toLucenePoint(point));
|
||||
// points are a special "shape" case: for queries we need to quantize since the Lucene
|
||||
// tessellator doesn't do anything with them.
|
||||
// todo this is a sandy lucene experience so we should investigate fixing this upstream
|
||||
double quantizedLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(point.getLat()));
|
||||
double quantizedLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(point.getLon()));
|
||||
geometries.add(new org.apache.lucene.geo.Point(quantizedLat, quantizedLon));
|
||||
}
|
||||
return null;
|
||||
|
||||
|
|
|
|||
|
|
@ -34,15 +34,17 @@ package org.opensearch.index.codec;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode;
|
||||
import org.apache.lucene.document.BinaryDocValuesField;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.index.SegmentReader;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
|
||||
import org.opensearch.common.settings.Settings;
|
||||
import org.opensearch.env.Environment;
|
||||
|
|
@ -72,16 +74,37 @@ public class CodecTests extends OpenSearchTestCase {
|
|||
|
||||
public void testDefault() throws Exception {
|
||||
Codec codec = createCodecService().codec("default");
|
||||
assertCompressionEquals(Mode.BEST_SPEED, codec);
|
||||
assertDVCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec);
|
||||
assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, codec);
|
||||
}
|
||||
|
||||
public void testBestCompression() throws Exception {
|
||||
Codec codec = createCodecService().codec("best_compression");
|
||||
assertCompressionEquals(Mode.BEST_COMPRESSION, codec);
|
||||
assertDVCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec);
|
||||
assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, codec);
|
||||
}
|
||||
|
||||
private void assertDVCompressionEquals(Lucene80DocValuesFormat.Mode expected, Codec actual) throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = newIndexWriterConfig(null);
|
||||
iwc.setCodec(actual);
|
||||
IndexWriter iw = new IndexWriter(dir, iwc);
|
||||
Document doc = new Document();
|
||||
doc.add(new BinaryDocValuesField("foo", new BytesRef("aaa")));
|
||||
iw.addDocument(doc);
|
||||
iw.commit();
|
||||
iw.close();
|
||||
DirectoryReader ir = DirectoryReader.open(dir);
|
||||
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
|
||||
String v = sr.getFieldInfos().fieldInfo("foo").getAttribute(Lucene80DocValuesFormat.MODE_KEY);
|
||||
assertNotNull(v);
|
||||
assertEquals(expected, Lucene80DocValuesFormat.Mode.valueOf(v));
|
||||
ir.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
// write some docs with it, inspect .si to see this was the used compression
|
||||
private void assertCompressionEquals(Mode expected, Codec actual) throws Exception {
|
||||
private void assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode expected, Codec actual) throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = newIndexWriterConfig(null);
|
||||
iwc.setCodec(actual);
|
||||
|
|
@ -93,7 +116,7 @@ public class CodecTests extends OpenSearchTestCase {
|
|||
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
|
||||
String v = sr.getSegmentInfo().info.getAttribute(Lucene87StoredFieldsFormat.MODE_KEY);
|
||||
assertNotNull(v);
|
||||
assertEquals(expected, Mode.valueOf(v));
|
||||
assertEquals(expected, Lucene87StoredFieldsFormat.Mode.valueOf(v));
|
||||
ir.close();
|
||||
dir.close();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
|
|||
// TODO: these charfilters are not yet exposed: useful?
|
||||
// handling of zwnj for persian
|
||||
.put("persian", Void.class)
|
||||
.put("cjkwidth", Void.class)
|
||||
.immutableMap();
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Reference in New Issue