Upgrade to lucene-8.5.0-snapshot-d62f6307658 (#52039) (#52130)

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.7.0
-lucene            = 8.5.0-snapshot-3333ce7da6d
+lucene            = 8.5.0-snapshot-d62f6307658
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 13.0.2+8

modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-f8ccecbafaae02529259336529be9c975df8f042

modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+81e4689ebebfc0d5bb7d18ac328a6c8864283fb6

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-dd52e75c39faeeae458675f42c1952554dbd14c6

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+66b51c3a54ac621a8bd94ce6a9de3bec9df9a39a

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-966d7f88e4ce01e70e092eeb1f73d46af415c61d

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+143c7e186fd571704b6ca60d06ec77e4f9fac368

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-396e3966f02dde019f269bf4478e7bb51330ca48

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+3b6b79e00b08d29bf5a85e1f99216a65695d9b6e

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-aeb18b5d73d6304cb77c53b44d29be9a12f37621

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+590d65061876a9ca7f3ea531048c9856187526aa

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-f080232d355acb6728f45a4bc400080edcd0a900

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+0cc3e11974dc58556094423d492b462a60d46dbc

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-fbe54b98515f263e1c60b20cb2178b401e03b633

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+941e7eb32b43709baf05bfd8fdf37cc89fa60afa

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-890953e7f21780dc3f2cc9b77cdbc96092bf5266

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+8df4618d73fa321f75daf43c7d4e976af95e5249

server/licenses/lucene-analyzers-common-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-3bfaed62b80d02bb697a7bd69e06441860d5ccda

server/licenses/lucene-analyzers-common-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+8e7e27a4361de0faffe9d75003b3ad27bd19cdc5

server/licenses/lucene-backward-codecs-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-e57bca7c4e596914d2eb2b7fedd06f24bffd02a9

server/licenses/lucene-backward-codecs-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+d619ec79dad9218b38d1eb766344393f335808dc

server/licenses/lucene-core-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-872d97891b63a07475f2ad54b5cd2532ca949b9f

server/licenses/lucene-core-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+bcb2c684915d642c9452298fc5a8ca96e6eb33d3

server/licenses/lucene-grouping-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-ab54d212db5d76d210d632648da23d659b95c23c

server/licenses/lucene-grouping-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+c14f6ad5ac0bc2e56148bb221695cbf74d237070

server/licenses/lucene-highlighter-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-81ded52ae9e7dfe7a494d1f0c4ae206d245e4f68

server/licenses/lucene-highlighter-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+f093650ac3ee0393b6498d4ac167d1177e7a07ec

server/licenses/lucene-join-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-09888a8ee3f294b64686e6ce1c838e4b715af3ff

server/licenses/lucene-join-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+88e2018c6a3bd340b60c4b9b9d20effa589cd1f3

server/licenses/lucene-memory-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-dc991eab909c597c357096ee8a8c1596d6336e4f

server/licenses/lucene-memory-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+04d36b02d9d5329e15684483410c5dc8941190a3

server/licenses/lucene-misc-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-2121b2867debc14ff5263fef0228bcbc00ecfdb3

server/licenses/lucene-misc-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+6d489d4769a6f6044f968543f763c33ab50d9eb8

server/licenses/lucene-queries-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-c3c4baa42396f44b4be1297e02530115fc77db0c

server/licenses/lucene-queries-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+caf369c56e13ba576555ab06be6d846f184bdc7d

server/licenses/lucene-queryparser-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-c9d2393c5587c3948ebfef2bbc864e49a42d64ae

server/licenses/lucene-queryparser-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+9fd37ee637c1fde869ed5730fba38a7920a33c30

server/licenses/lucene-sandbox-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-76f51eb693a3befa3ff56cca16b111bcaa84a5f9

server/licenses/lucene-sandbox-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+7122730e99902ba5394366abccd14e70b64f501a

server/licenses/lucene-spatial-extras-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-3fbd17d1ba11874cbf6dfb766fd41e4ac1ed1289

server/licenses/lucene-spatial-extras-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+d3a1d0a426571ad7c27118dde28053e0c269cd83

server/licenses/lucene-spatial3d-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-bf7aad63c08231f24e9108eb7262cd1950a96158

server/licenses/lucene-spatial3d-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+c23c462831d13ae2b770acf232c83ce8627bbfdc

server/licenses/lucene-suggest-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-5e24487295d4c927e75d3ea0b7f6724ee9043b80

server/licenses/lucene-suggest-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+444656499dcfcacc85cb4f20e32868c9d6ffc61f

server/src/main/java/org/apache/lucene/queries/XIntervals.java

@@ -25,6 +25,7 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.queries.intervals.IntervalIterator;
+import org.apache.lucene.queries.intervals.IntervalMatchesIterator;
 import org.apache.lucene.queries.intervals.IntervalQuery;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
@@ -111,7 +112,7 @@ public final class XIntervals {
         }
 
         @Override
-        public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
+        public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
             Terms terms = ctx.reader().terms(field);
             if (terms == null) {
                 return null;
@@ -129,7 +130,56 @@ public final class XIntervals {
                     }
                 }
             }
-            return MatchesUtils.disjunction(subMatches);
+            MatchesIterator mi = MatchesUtils.disjunction(subMatches);
+            if (mi == null) {
+                return null;
+            }
+            return new IntervalMatchesIterator() {
+                @Override
+                public int gaps() {
+                    return 0;
+                }
+
+                @Override
+                public int width() {
+                    return 1;
+                }
+
+                @Override
+                public boolean next() throws IOException {
+                    return mi.next();
+                }
+
+                @Override
+                public int startPosition() {
+                    return mi.startPosition();
+                }
+
+                @Override
+                public int endPosition() {
+                    return mi.endPosition();
+                }
+
+                @Override
+                public int startOffset() throws IOException {
+                    return mi.startOffset();
+                }
+
+                @Override
+                public int endOffset() throws IOException {
+                    return mi.endOffset();
+                }
+
+                @Override
+                public MatchesIterator getSubMatches() throws IOException {
+                    return mi.getSubMatches();
+                }
+
+                @Override
+                public Query getQuery() {
+                    return mi.getQuery();
+                }
+            };
         }
 
         @Override
@@ -684,7 +734,7 @@ public final class XIntervals {
         }
 
         @Override
-        public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
+        public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
             Terms terms = ctx.reader().terms(field);
             if (terms == null)
                 return null;
@@ -699,12 +749,22 @@ public final class XIntervals {
             return matches(te, doc);
         }
 
-        static MatchesIterator matches(TermsEnum te, int doc) throws IOException {
+        static IntervalMatchesIterator matches(TermsEnum te, int doc) throws IOException {
             PostingsEnum pe = te.postings(null, PostingsEnum.OFFSETS);
             if (pe.advance(doc) != doc) {
                 return null;
             }
-            return new MatchesIterator() {
+            return new IntervalMatchesIterator() {
+
+                @Override
+                public int gaps() {
+                    return 0;
+                }
+
+                @Override
+                public int width() {
+                    return 1;
+                }
 
                 int upto = pe.freq();
                 int pos = -1;

server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java

@@ -26,8 +26,8 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.intervals.IntervalMatchesIterator;
 import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.MatchesIterator;
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.queries.intervals.IntervalIterator;
 import org.apache.lucene.queries.intervals.Intervals;
@@ -284,7 +284,7 @@ public class IntervalBuilder {
         }
 
         @Override
-        public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) {
+        public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) {
             return null;
         }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java

@@ -44,6 +44,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -51,9 +52,14 @@ import java.util.Map;
 import java.util.function.Supplier;
 
 import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_EXTENSION;
-import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_INDEX_EXTENSION;
+import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.INDEX_EXTENSION_PREFIX;
+import static org.apache.lucene.codecs.compressing.FieldsIndexWriter.FIELDS_INDEX_EXTENSION_SUFFIX;
+import static org.apache.lucene.codecs.compressing.FieldsIndexWriter.FIELDS_META_EXTENSION_SUFFIX;
 
 public class SourceOnlySnapshot {
+
+    private static final String FIELDS_INDEX_EXTENSION = INDEX_EXTENSION_PREFIX + FIELDS_INDEX_EXTENSION_SUFFIX;
+    private static final String FIELDS_META_EXTENSION = INDEX_EXTENSION_PREFIX + FIELDS_META_EXTENSION_SUFFIX;
     private final Directory targetDirectory;
     private final Supplier<Query> deleteByQuerySupplier;
 
@@ -208,12 +214,16 @@ public class SourceOnlySnapshot {
             newInfo.setFieldInfosFiles(trackingDir.getCreatedFiles());
             String idxFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_INDEX_EXTENSION);
             String dataFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_EXTENSION);
+            String metaFile = IndexFileNames.segmentFileName(newSegmentInfo.name, segmentSuffix, FIELDS_META_EXTENSION);
             Directory sourceDir = newSegmentInfo.dir;
             if (si.getUseCompoundFile()) {
                 sourceDir = codec.compoundFormat().getCompoundReader(sourceDir, si, IOContext.DEFAULT);
             }
             trackingDir.copyFrom(sourceDir, idxFile, idxFile, IOContext.DEFAULT);
             trackingDir.copyFrom(sourceDir, dataFile, dataFile, IOContext.DEFAULT);
+             if (Arrays.asList(sourceDir.listAll()).contains(metaFile)) { // only exists for Lucene 8.5+ indices
+                 trackingDir.copyFrom(sourceDir, metaFile, metaFile, IOContext.DEFAULT);
+             }
             if (sourceDir != newSegmentInfo.dir) {
                 sourceDir.close();
             }

x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java

@@ -148,10 +148,10 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase {
                 snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, Collections.emptyMap(), future));
             shardGeneration = future.actionGet();
             IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
-            // we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt
+            // we processed the segments_N file plus _1.si, _1.fnm, _1.fdx, _1.fdt, _1.fdm
-            assertEquals(5, copy.getIncrementalFileCount());
+            assertEquals(6, copy.getIncrementalFileCount());
-            // in total we have 4 more files than the previous snap since we don't count the segments_N twice
+            // in total we have 5 more files than the previous snap since we don't count the segments_N twice
-            assertEquals(totalFileCount+4, copy.getTotalFileCount());
+            assertEquals(totalFileCount+5, copy.getTotalFileCount());
             assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE);
         }
         deleteDoc(shard, "_doc", Integer.toString(10));
@@ -166,8 +166,8 @@ public class SourceOnlySnapshotShardTests extends IndexShardTestCase {
             IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy();
             // we processed the segments_N file plus _1_1.liv
             assertEquals(2, copy.getIncrementalFileCount());
-            // in total we have 5 more files than the previous snap since we don't count the segments_N twice
+            // in total we have 6 more files than the previous snap since we don't count the segments_N twice
-            assertEquals(totalFileCount+5, copy.getTotalFileCount());
+            assertEquals(totalFileCount+6, copy.getTotalFileCount());
             assertEquals(copy.getStage(), IndexShardSnapshotStatus.Stage.DONE);
         }
         closeShards(shard);

x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotTests.java

@@ -204,12 +204,13 @@ public class SourceOnlySnapshotTests extends ESTestCase {
                 snapshot = deletionPolicy.snapshot();
                 snapshoter = new SourceOnlySnapshot(targetDir);
                 createdFiles = snapshoter.syncSnapshot(snapshot);
-                assertEquals(4, createdFiles.size());
+                assertEquals(5, createdFiles.size());
                 for (String file : createdFiles) {
                     String extension = IndexFileNames.getExtension(file);
                     switch (extension) {
                         case "fdt":
                         case "fdx":
+                        case "fdm":
                         case "fnm":
                         case "si":
                             break;

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -1 +0,0 @@
-872d97891b63a07475f2ad54b5cd2532ca949b9f

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-d62f6307658.jar.sha1

@@ -0,0 +1 @@
+bcb2c684915d642c9452298fc5a8ca96e6eb33d3

x-pack/plugin/src/test/resources/rest-api-spec/test/snapshot/10_basic.yml

@@ -69,7 +69,9 @@ setup:
   - match: { test_index.shards.0.translog.recovered: 0}
   - match: { test_index.shards.0.translog.total: 0}
   - match: { test_index.shards.0.translog.total_on_start: 0}
-  - match: { test_index.shards.0.index.files.recovered: 5}
+  # for version >= 7.7, there should be 6, previous versions should have 5
+  - gte: { test_index.shards.0.index.files.recovered: 5}
+  - lte: { test_index.shards.0.index.files.recovered: 6}
   - match: { test_index.shards.0.index.files.reused: 0}
   - match: { test_index.shards.0.index.size.reused_in_bytes: 0}
   - gt: { test_index.shards.0.index.size.recovered_in_bytes: 0}