honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade to lucene-8.5.0-snapshot-d62f6307658 (#52039) (#52130)

This commit is contained in:
Ignacio Vera 2020-02-10 10:13:22 +01:00 committed by GitHub
parent 1cefafdd14
commit 80e3c97210
53 changed files with 113 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/version.properties
View File

@ -1,5 +1,5 @@
elasticsearch = 7.7.0
lucene = 8.5.0-snapshot-3333ce7da6d
lucene = 8.5.0-snapshot-d62f6307658
bundled_jdk_vendor = adoptopenjdk
bundled_jdk = 13.0.2+8

1
modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
f8ccecbafaae02529259336529be9c975df8f042

1
modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
81e4689ebebfc0d5bb7d18ac328a6c8864283fb6

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
dd52e75c39faeeae458675f42c1952554dbd14c6

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
66b51c3a54ac621a8bd94ce6a9de3bec9df9a39a

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
966d7f88e4ce01e70e092eeb1f73d46af415c61d

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
143c7e186fd571704b6ca60d06ec77e4f9fac368

1
plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
396e3966f02dde019f269bf4478e7bb51330ca48

1
plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
3b6b79e00b08d29bf5a85e1f99216a65695d9b6e

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
aeb18b5d73d6304cb77c53b44d29be9a12f37621

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
590d65061876a9ca7f3ea531048c9856187526aa

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
f080232d355acb6728f45a4bc400080edcd0a900

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
0cc3e11974dc58556094423d492b462a60d46dbc

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
fbe54b98515f263e1c60b20cb2178b401e03b633

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
941e7eb32b43709baf05bfd8fdf37cc89fa60afa

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
890953e7f21780dc3f2cc9b77cdbc96092bf5266

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
8df4618d73fa321f75daf43c7d4e976af95e5249

1
server/licenses/lucene-analyzers-common-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
3bfaed62b80d02bb697a7bd69e06441860d5ccda

1
server/licenses/lucene-analyzers-common-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
8e7e27a4361de0faffe9d75003b3ad27bd19cdc5

1
server/licenses/lucene-backward-codecs-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
e57bca7c4e596914d2eb2b7fedd06f24bffd02a9

1
server/licenses/lucene-backward-codecs-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d619ec79dad9218b38d1eb766344393f335808dc

1
server/licenses/lucene-core-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
872d97891b63a07475f2ad54b5cd2532ca949b9f

1
server/licenses/lucene-core-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
bcb2c684915d642c9452298fc5a8ca96e6eb33d3

1
server/licenses/lucene-grouping-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
ab54d212db5d76d210d632648da23d659b95c23c

1
server/licenses/lucene-grouping-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
c14f6ad5ac0bc2e56148bb221695cbf74d237070

1
server/licenses/lucene-highlighter-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
81ded52ae9e7dfe7a494d1f0c4ae206d245e4f68

1
server/licenses/lucene-highlighter-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
f093650ac3ee0393b6498d4ac167d1177e7a07ec

1
server/licenses/lucene-join-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
09888a8ee3f294b64686e6ce1c838e4b715af3ff

1
server/licenses/lucene-join-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
88e2018c6a3bd340b60c4b9b9d20effa589cd1f3

1
server/licenses/lucene-memory-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
dc991eab909c597c357096ee8a8c1596d6336e4f

1
server/licenses/lucene-memory-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
04d36b02d9d5329e15684483410c5dc8941190a3

1
server/licenses/lucene-misc-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
2121b2867debc14ff5263fef0228bcbc00ecfdb3

1
server/licenses/lucene-misc-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
6d489d4769a6f6044f968543f763c33ab50d9eb8

1
server/licenses/lucene-queries-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
c3c4baa42396f44b4be1297e02530115fc77db0c

1
server/licenses/lucene-queries-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
caf369c56e13ba576555ab06be6d846f184bdc7d

1
server/licenses/lucene-queryparser-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
c9d2393c5587c3948ebfef2bbc864e49a42d64ae

1
server/licenses/lucene-queryparser-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
9fd37ee637c1fde869ed5730fba38a7920a33c30

1
server/licenses/lucene-sandbox-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
76f51eb693a3befa3ff56cca16b111bcaa84a5f9

1
server/licenses/lucene-sandbox-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7122730e99902ba5394366abccd14e70b64f501a

1
server/licenses/lucene-spatial-extras-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
3fbd17d1ba11874cbf6dfb766fd41e4ac1ed1289

1
server/licenses/lucene-spatial-extras-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d3a1d0a426571ad7c27118dde28053e0c269cd83

1
server/licenses/lucene-spatial3d-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
bf7aad63c08231f24e9108eb7262cd1950a96158

1
server/licenses/lucene-spatial3d-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
c23c462831d13ae2b770acf232c83ce8627bbfdc

1
server/licenses/lucene-suggest-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
5e24487295d4c927e75d3ea0b7f6724ee9043b80

1
server/licenses/lucene-suggest-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
444656499dcfcacc85cb4f20e32868c9d6ffc61f

70
server/src/main/java/org/apache/lucene/queries/XIntervals.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queries.intervals.IntervalIterator;
import org.apache.lucene.queries.intervals.IntervalMatchesIterator;
import org.apache.lucene.queries.intervals.IntervalQuery;
import org.apache.lucene.queries.intervals.Intervals;
import org.apache.lucene.queries.intervals.IntervalsSource;
@ -111,7 +112,7 @@ public final class XIntervals {
}
@Override
public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
Terms terms = ctx.reader().terms(field);
if (terms == null) {
return null;
@ -129,7 +130,56 @@ public final class XIntervals {
}
}
}
return MatchesUtils.disjunction(subMatches);
MatchesIterator mi = MatchesUtils.disjunction(subMatches);
if (mi == null) {
return null;
}
return new IntervalMatchesIterator() {
@Override
public int gaps() {
return 0;
}
@Override
public int width() {
return 1;
}
@Override
public boolean next() throws IOException {
return mi.next();
}
@Override
public int startPosition() {
return mi.startPosition();
}
@Override
public int endPosition() {
return mi.endPosition();
}
@Override
public int startOffset() throws IOException {
return mi.startOffset();
}
@Override
public int endOffset() throws IOException {
return mi.endOffset();
}
@Override
public MatchesIterator getSubMatches() throws IOException {
return mi.getSubMatches();
}
@Override
public Query getQuery() {
return mi.getQuery();
}
};
}
@Override
@ -684,7 +734,7 @@ public final class XIntervals {
}
@Override
public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
Terms terms = ctx.reader().terms(field);
if (terms == null)
return null;
@ -699,12 +749,22 @@ public final class XIntervals {
return matches(te, doc);
}
static MatchesIterator matches(TermsEnum te, int doc) throws IOException {
static IntervalMatchesIterator matches(TermsEnum te, int doc) throws IOException {
PostingsEnum pe = te.postings(null, PostingsEnum.OFFSETS);
if (pe.advance(doc) != doc) {
return null;
}
return new MatchesIterator() {
return new IntervalMatchesIterator() {
@Override
public int gaps() {
return 0;
}
@Override
public int width() {
return 1;
}
int upto = pe.freq();
int pos = -1;

4
server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java
View File

@ -26,8 +26,8 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.intervals.IntervalMatchesIterator;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.MatchesIterator;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.queries.intervals.IntervalIterator;
import org.apache.lucene.queries.intervals.Intervals;
@ -284,7 +284,7 @@ public class IntervalBuilder {
}
@Override
public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) {
public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) {
return null;
}

12
x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
View File

@ -44,6 +44,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -51,9 +52,14 @@ import java.util.Map;
import java.util.function.Supplier;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_EXTENSION;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_INDEX_EXTENSION;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.INDEX_EXTENSION_PREFIX;
import static org.apache.lucene.codecs.compressing.FieldsIndexWriter.FIELDS_INDEX_EXTENSION_SUFFIX;
import static org.apache.lucene.codecs.compressing.FieldsIndexWriter.FIELDS_META_EXTENSION_SUFFIX;
public class SourceOnlySnapshot {
private static final String FIELDS_INDEX_EXTENSION = INDEX_EXTENSION_PREFIX + FIELDS_INDEX_EXTENSION_SUFFIX;
private static final String FIELDS_META_EXTENSION = INDEX_EXTENSION_PREFIX + FIELDS_META_EXTENSION_SUFFIX;
private final Directory targetDirectory;
private final Supplier<Query> deleteByQuerySupplier;
@ -208,12 +214,16 @@ public class SourceOnlySnapshot {
newInfo.setFieldInfosFiles(trackingDir.getCreatedFiles());
String idxFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_INDEX_EXTENSION);
String dataFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_EXTENSION);
String metaFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_META_EXTENSION);
Directory sourceDir = newSegmentInfo.dir;
if (si.getUseCompoundFile()) {
sourceDir = codec.compoundFormat().getCompoundReader(sourceDir, si, IOContext.DEFAULT);
}
trackingDir.copyFrom(sourceDir, idxFile, idxFile, IOContext.DEFAULT);
trackingDir.copyFrom(sourceDir, dataFile, dataFile, IOContext.DEFAULT);
if (Arrays.asList(sourceDir.listAll()).contains(metaFile)) { // only exists for Lucene 8.5+ indices
trackingDir.copyFrom(sourceDir, metaFile, metaFile, IOContext.DEFAULT);
}
if (sourceDir != newSegmentInfo.dir) {
sourceDir.close();
}

12
x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
View File

@ -148,10 +148,10 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase {
snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, Collections.emptyMap(), future));
shardGeneration = future.actionGet();
IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
// we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt
assertEquals(5, copy.getIncrementalFileCount());
// in total we have 4 more files than the previous snap since we don't count the segments_N twice
assertEquals(totalFileCount+4, copy.getTotalFileCount());
// we processed the segments_N file plus _1.si, _1.fnm, _1.fdx, _1.fdt, _1.fdm
assertEquals(6, copy.getIncrementalFileCount());
// in total we have 5 more files than the previous snap since we don't count the segments_N twice
assertEquals(totalFileCount+5, copy.getTotalFileCount());
assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE);
}
deleteDoc(shard, "_doc", Integer.toString(10));
@ -166,8 +166,8 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase {
IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
// we processed the segments_N file plus _1_1.liv
assertEquals(2, copy.getIncrementalFileCount());
// in total we have 5 more files than the previous snap since we don't count the segments_N twice
assertEquals(totalFileCount+5, copy.getTotalFileCount());
// in total we have 6 more files than the previous snap since we don't count the segments_N twice
assertEquals(totalFileCount+6, copy.getTotalFileCount());
assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE);
}
closeShards(shard);

3
x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java
View File

@ -204,12 +204,13 @@ public class SourceOnlySnapshotTests extends ESTestCase {
snapshot = deletionPolicy.snapshot();
snapshoter = new SourceOnlySnapshot(targetDir);
createdFiles = snapshoter.syncSnapshot(snapshot);
assertEquals(4, createdFiles.size());
assertEquals(5, createdFiles.size());
for (String file : createdFiles) {
String extension = IndexFileNames.getExtension(file);
switch (extension) {
case "fdt":
case "fdx":
case "fdm":
case "fnm":
case "si":
break;

1
x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-3333ce7da6d.jar.sha1
View File

@ -1 +0,0 @@
872d97891b63a07475f2ad54b5cd2532ca949b9f

1
x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-d62f6307658.jar.sha1 Normal file
View File

@ -0,0 +1 @@
bcb2c684915d642c9452298fc5a8ca96e6eb33d3

4
x-pack/plugin/src/test/resources/rest-api-spec/test/snapshot/10_basic.yml
View File

@ -69,7 +69,9 @@ setup:
- match: { test_index.shards.0.translog.recovered: 0}
- match: { test_index.shards.0.translog.total: 0}
- match: { test_index.shards.0.translog.total_on_start: 0}
- match: { test_index.shards.0.index.files.recovered: 5}
# for version >= 7.7, there should be 6, previous versions should have 5
- gte: { test_index.shards.0.index.files.recovered: 5}
- lte: { test_index.shards.0.index.files.recovered: 6}
- match: { test_index.shards.0.index.files.reused: 0}
- match: { test_index.shards.0.index.size.reused_in_bytes: 0}
- gt: { test_index.shards.0.index.size.recovered_in_bytes: 0}