Upgrade to lucene-8.5.0-snapshot-3333ce7da6d (#51749)

Backport for #51327

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.7.0
-lucene            = 8.4.0
+lucene            = 8.5.0-snapshot-3333ce7da6d
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 13.0.2+8

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        8.4.0
+:lucene_version:        8.5.0
-:lucene_version_path:   8_4_0
+:lucene_version_path:   8_5_0
 :jdk:                   1.8.0_131
 :jdk_major:             8
 :build_flavor:          default

modules/lang-expression/licenses/lucene-expressions-8.4.0.jar.sha1

@@ -1 +0,0 @@
-2aeb62a1a6278c2707b9a491c74a102c1f11f06e

modules/lang-expression/licenses/lucene-expressions-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+f8ccecbafaae02529259336529be9c975df8f042

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.4.0.jar.sha1

@@ -1 +0,0 @@
-8e02317f9253512b0d4fc8d5c41e2b8c647955d2

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+dd52e75c39faeeae458675f42c1952554dbd14c6

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.4.0.jar.sha1

@@ -1 +0,0 @@
-5ccb38f67ac1895d25c7c84abe8805aef5526790

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+966d7f88e4ce01e70e092eeb1f73d46af415c61d

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.4.0.jar.sha1

@@ -1 +0,0 @@
-7f57aff713c474ffb6aba1bae6d9374a1a184114

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+396e3966f02dde019f269bf4478e7bb51330ca48

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.4.0.jar.sha1

@@ -1 +0,0 @@
-e47cb4efd4e9e071e0563dd44f1759ea1e712ca7

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+aeb18b5d73d6304cb77c53b44d29be9a12f37621

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.4.0.jar.sha1

@@ -1 +0,0 @@
-39a8ecf0d875630d04972208e00adf930500f52b

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+f080232d355acb6728f45a4bc400080edcd0a900

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.4.0.jar.sha1

@@ -1 +0,0 @@
-c9815fc970936940ab08880e0d34c8cff169ee4d

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+fbe54b98515f263e1c60b20cb2178b401e03b633

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.4.0.jar.sha1

@@ -1 +0,0 @@
-f389d3873ee33b5dd432077e201a541c7a16772a

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+890953e7f21780dc3f2cc9b77cdbc96092bf5266

server/build.gradle

@@ -96,7 +96,6 @@ dependencies {
   compile "org.apache.lucene:lucene-queries:${versions.lucene}"
   compile "org.apache.lucene:lucene-queryparser:${versions.lucene}"
   compile "org.apache.lucene:lucene-sandbox:${versions.lucene}"
-  compile "org.apache.lucene:lucene-spatial:${versions.lucene}"
   compile "org.apache.lucene:lucene-spatial-extras:${versions.lucene}"
   compile "org.apache.lucene:lucene-spatial3d:${versions.lucene}"
   compile "org.apache.lucene:lucene-suggest:${versions.lucene}"

server/licenses/lucene-analyzers-common-8.4.0.jar.sha1

@@ -1 +0,0 @@
-79d7792942fa009316de2d7d1a4d7e8b33548947

server/licenses/lucene-analyzers-common-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+3bfaed62b80d02bb697a7bd69e06441860d5ccda

server/licenses/lucene-backward-codecs-8.4.0.jar.sha1

@@ -1 +0,0 @@
-6604030f7da573a8c00641f9c7deef6c143b6022

server/licenses/lucene-backward-codecs-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+e57bca7c4e596914d2eb2b7fedd06f24bffd02a9

server/licenses/lucene-core-8.4.0.jar.sha1

@@ -1 +0,0 @@
-2746f9ec96f9ce3a345b11f03751136073f7869f

server/licenses/lucene-core-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+872d97891b63a07475f2ad54b5cd2532ca949b9f

server/licenses/lucene-grouping-8.4.0.jar.sha1

@@ -1 +0,0 @@
-f73773fc39d43df7661609b9f7a733ddfd091af7

server/licenses/lucene-grouping-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+ab54d212db5d76d210d632648da23d659b95c23c

server/licenses/lucene-highlighter-8.4.0.jar.sha1

@@ -1 +0,0 @@
-be8a20787124cf52c56c5928ef970df2d8a26f51

server/licenses/lucene-highlighter-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+81ded52ae9e7dfe7a494d1f0c4ae206d245e4f68

server/licenses/lucene-join-8.4.0.jar.sha1

@@ -1 +0,0 @@
-8ec1dce97ba5b616e165068225bba873179482e9

server/licenses/lucene-join-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+09888a8ee3f294b64686e6ce1c838e4b715af3ff

server/licenses/lucene-memory-8.4.0.jar.sha1

@@ -1 +0,0 @@
-dff1e225fe6bfdf7853663bc48831e9714bf035e

server/licenses/lucene-memory-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+dc991eab909c597c357096ee8a8c1596d6336e4f

server/licenses/lucene-misc-8.4.0.jar.sha1

@@ -1 +0,0 @@
-ebe2549568386d5c289ec0eb738172f1a0445259

server/licenses/lucene-misc-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+2121b2867debc14ff5263fef0228bcbc00ecfdb3

server/licenses/lucene-queries-8.4.0.jar.sha1

@@ -1 +0,0 @@
-9d5fdc88f91586bf5d1afa13b9a77302c39b5e7c

server/licenses/lucene-queries-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+c3c4baa42396f44b4be1297e02530115fc77db0c

server/licenses/lucene-queryparser-8.4.0.jar.sha1

@@ -1 +0,0 @@
-ab7c7c3c823cb2f8fb1b54fdc82b3e133e8e8344

server/licenses/lucene-queryparser-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+c9d2393c5587c3948ebfef2bbc864e49a42d64ae

server/licenses/lucene-sandbox-8.4.0.jar.sha1

@@ -1 +0,0 @@
-bc34429b8d1a620c58639f376bee9ba425a035d3

server/licenses/lucene-sandbox-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+76f51eb693a3befa3ff56cca16b111bcaa84a5f9

server/licenses/lucene-spatial-8.4.0.jar.sha1

@@ -1 +0,0 @@
-0bc8d9f00bd34806bc82d01390855ef9dcbea85b

server/licenses/lucene-spatial-extras-8.4.0.jar.sha1

@@ -1 +0,0 @@
-a1e978879d35af3590549437b80679b5c00f27d6

server/licenses/lucene-spatial-extras-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+3fbd17d1ba11874cbf6dfb766fd41e4ac1ed1289

server/licenses/lucene-spatial3d-8.4.0.jar.sha1

@@ -1 +0,0 @@
-1000c919125bb13f265b101341c34bb5af814fd3

server/licenses/lucene-spatial3d-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+bf7aad63c08231f24e9108eb7262cd1950a96158

server/licenses/lucene-suggest-8.4.0.jar.sha1

@@ -1 +0,0 @@
-488e5cfdd4d2d30b161fc45819a82a6984eb0f99

server/licenses/lucene-suggest-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+5e24487295d4c927e75d3ea0b7f6724ee9043b80

server/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java

@@ -21,6 +21,11 @@ package org.apache.lucene.search.uhighlight;
 
 import java.text.BreakIterator;
 import java.util.Locale;
+import java.util.PriorityQueue;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
 
 import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR;
 
@@ -76,4 +81,84 @@ class CustomFieldHighlighter extends FieldHighlighter {
         }
         return EMPTY_PASSAGE;
     }
+
+    // TODO: use FieldHighlighter::highlightOffsetsEnums and modify BoundedBreakIteratorScanner to work with it
+    // LUCENE-9093 modified how FieldHighlighter breaks texts into passages,
+    // which doesn't work well with BoundedBreakIteratorScanner
+    // This is the copy of highlightOffsetsEnums before LUCENE-9093.
+    @Override
+    protected Passage[] highlightOffsetsEnums(OffsetsEnum off)
+        throws IOException {
+
+        final int contentLength = this.breakIterator.getText().getEndIndex();
+
+        if (off.nextPosition() == false) {
+            return new Passage[0];
+        }
+
+        PriorityQueue<Passage> passageQueue = new PriorityQueue<>(Math.min(64, maxPassages + 1), (left, right) -> {
+            if (left.getScore() < right.getScore()) {
+                return -1;
+            } else if (left.getScore() > right.getScore()) {
+                return 1;
+            } else {
+                return left.getStartOffset() - right.getStartOffset();
+            }
+        });
+        Passage passage = new Passage(); // the current passage in-progress.  Will either get reset or added to queue.
+
+        do {
+            int start = off.startOffset();
+            if (start == -1) {
+                throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
+            }
+            int end = off.endOffset();
+            if (start < contentLength && end > contentLength) {
+                continue;
+            }
+            // See if this term should be part of a new passage.
+            if (start >= passage.getEndOffset()) {
+                passage = maybeAddPassage(passageQueue, passageScorer, passage, contentLength);
+                // if we exceed limit, we are done
+                if (start >= contentLength) {
+                    break;
+                }
+                passage.setStartOffset(Math.max(this.breakIterator.preceding(start + 1), 0));
+                passage.setEndOffset(Math.min(this.breakIterator.following(start), contentLength));
+            }
+            // Add this term to the passage.
+            BytesRef term = off.getTerm();// a reference; safe to refer to
+            assert term != null;
+            passage.addMatch(start, end, term, off.freq());
+        } while (off.nextPosition());
+        maybeAddPassage(passageQueue, passageScorer, passage, contentLength);
+
+        Passage[] passages = passageQueue.toArray(new Passage[passageQueue.size()]);
+        // sort in ascending order
+        Arrays.sort(passages, Comparator.comparingInt(Passage::getStartOffset));
+        return passages;
+    }
+
+    // TODO: use FieldHighlighter::maybeAddPassage
+    // After removing CustomFieldHighlighter::highlightOffsetsEnums, remove this method as well.
+    private Passage maybeAddPassage(PriorityQueue<Passage> passageQueue, PassageScorer scorer, Passage passage, int contentLength) {
+        if (passage.getStartOffset() == -1) {
+            // empty passage, we can ignore it
+            return passage;
+        }
+        passage.setScore(scorer.score(passage, contentLength));
+        // new sentence: first add 'passage' to queue
+        if (passageQueue.size() == maxPassages && passage.getScore() < passageQueue.peek().getScore()) {
+            passage.reset(); // can't compete, just reset it
+        } else {
+            passageQueue.offer(passage);
+            if (passageQueue.size() > maxPassages) {
+                passage = passageQueue.poll();
+                passage.reset();
+            } else {
+                passage = new Passage();
+            }
+        }
+        return passage;
+    }
 }

server/src/main/java/org/elasticsearch/Version.java

@@ -129,7 +129,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_7_5_2 = new Version(7050299, org.apache.lucene.util.Version.LUCENE_8_3_0);
     public static final Version V_7_5_3 = new Version(7050399, org.apache.lucene.util.Version.LUCENE_8_3_0);
     public static final Version V_7_6_0 = new Version(7060099, org.apache.lucene.util.Version.LUCENE_8_4_0);
-    public static final Version V_7_7_0 = new Version(7070099, org.apache.lucene.util.Version.LUCENE_8_4_0);
+    public static final Version V_7_7_0 = new Version(7070099, org.apache.lucene.util.Version.LUCENE_8_5_0);
     public static final Version CURRENT = V_7_7_0;
 
     private static final ImmutableOpenIntMap<Version> idToVersion;

server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java

@@ -797,8 +797,8 @@ public abstract class IntervalsSourceProvider implements NamedWriteable, ToXCont
             BytesRef normalizedTerm = analyzer.normalize(fieldType.name(), term);
             FuzzyQuery fq = new FuzzyQuery(new Term(fieldType.name(), normalizedTerm),
                 fuzziness.asDistance(term), prefixLength, 128, transpositions);
-            CompiledAutomaton ca = new CompiledAutomaton(fq.toAutomaton());
+            CompiledAutomaton[] automata = fq.getAutomata();
-            source = XIntervals.multiterm(ca, term);
+            source = XIntervals.multiterm(automata[automata.length - 1], term);
             if (useField != null) {
                 source = Intervals.fixField(useField, source);
             }

server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java

@@ -550,7 +550,8 @@ public class IntervalQueryBuilderTests extends AbstractQueryTestCase<IntervalQue
 
     private static IntervalsSource buildFuzzySource(String term, String label, int prefixLength, boolean transpositions, int editDistance) {
         FuzzyQuery fq = new FuzzyQuery(new Term("field", term), editDistance, prefixLength, 128, transpositions);
-        return XIntervals.multiterm(new CompiledAutomaton(fq.toAutomaton()), label);
+        CompiledAutomaton[] automata = fq.getAutomata();
+        return XIntervals.multiterm(automata[automata.length - 1], label);
     }
 
     public void testFuzzy() throws IOException {

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.4.0.jar.sha1

@@ -1 +0,0 @@
-2746f9ec96f9ce3a345b11f03751136073f7869f

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.5.0-snapshot-3333ce7da6d.jar.sha1

@@ -0,0 +1 @@
+872d97891b63a07475f2ad54b5cd2532ca949b9f