Upgrade to lucene-8.5.0-snapshot-3333ce7da6d (#51749)
Backport for #51327
This commit is contained in:
Mayya Sharipova
GitHub
parent
1a9e7b2d69
commit
42b885f050
|
|
@ -1,5 +1,5 @@
|
|||
elasticsearch = 7.7.0
|
||||
lucene = 8.4.0
|
||||
lucene = 8.5.0-snapshot-3333ce7da6d
|
||||
|
||||
bundled_jdk_vendor = adoptopenjdk
|
||||
bundled_jdk = 13.0.2+8
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
|
||||
include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[]
|
||||
|
||||
:lucene_version: 8.4.0
|
||||
:lucene_version_path: 8_4_0
|
||||
:lucene_version: 8.5.0
|
||||
:lucene_version_path: 8_5_0
|
||||
:jdk: 1.8.0_131
|
||||
:jdk_major: 8
|
||||
:build_flavor: default
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
2aeb62a1a6278c2707b9a491c74a102c1f11f06e
|
||||
|
|
@ -0,0 +1 @@
|
|||
f8ccecbafaae02529259336529be9c975df8f042
|
||||
|
|
@ -1 +0,0 @@
|
|||
8e02317f9253512b0d4fc8d5c41e2b8c647955d2
|
||||
|
|
@ -0,0 +1 @@
|
|||
dd52e75c39faeeae458675f42c1952554dbd14c6
|
||||
|
|
@ -1 +0,0 @@
|
|||
5ccb38f67ac1895d25c7c84abe8805aef5526790
|
||||
|
|
@ -0,0 +1 @@
|
|||
966d7f88e4ce01e70e092eeb1f73d46af415c61d
|
||||
|
|
@ -1 +0,0 @@
|
|||
7f57aff713c474ffb6aba1bae6d9374a1a184114
|
||||
|
|
@ -0,0 +1 @@
|
|||
396e3966f02dde019f269bf4478e7bb51330ca48
|
||||
|
|
@ -1 +0,0 @@
|
|||
e47cb4efd4e9e071e0563dd44f1759ea1e712ca7
|
||||
|
|
@ -0,0 +1 @@
|
|||
aeb18b5d73d6304cb77c53b44d29be9a12f37621
|
||||
|
|
@ -1 +0,0 @@
|
|||
39a8ecf0d875630d04972208e00adf930500f52b
|
||||
|
|
@ -0,0 +1 @@
|
|||
f080232d355acb6728f45a4bc400080edcd0a900
|
||||
|
|
@ -1 +0,0 @@
|
|||
c9815fc970936940ab08880e0d34c8cff169ee4d
|
||||
|
|
@ -0,0 +1 @@
|
|||
fbe54b98515f263e1c60b20cb2178b401e03b633
|
||||
|
|
@ -1 +0,0 @@
|
|||
f389d3873ee33b5dd432077e201a541c7a16772a
|
||||
|
|
@ -0,0 +1 @@
|
|||
890953e7f21780dc3f2cc9b77cdbc96092bf5266
|
||||
|
|
@ -96,7 +96,6 @@ dependencies {
|
|||
compile "org.apache.lucene:lucene-queries:${versions.lucene}"
|
||||
compile "org.apache.lucene:lucene-queryparser:${versions.lucene}"
|
||||
compile "org.apache.lucene:lucene-sandbox:${versions.lucene}"
|
||||
compile "org.apache.lucene:lucene-spatial:${versions.lucene}"
|
||||
compile "org.apache.lucene:lucene-spatial-extras:${versions.lucene}"
|
||||
compile "org.apache.lucene:lucene-spatial3d:${versions.lucene}"
|
||||
compile "org.apache.lucene:lucene-suggest:${versions.lucene}"
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
79d7792942fa009316de2d7d1a4d7e8b33548947
|
||||
|
|
@ -0,0 +1 @@
|
|||
3bfaed62b80d02bb697a7bd69e06441860d5ccda
|
||||
|
|
@ -1 +0,0 @@
|
|||
6604030f7da573a8c00641f9c7deef6c143b6022
|
||||
|
|
@ -0,0 +1 @@
|
|||
e57bca7c4e596914d2eb2b7fedd06f24bffd02a9
|
||||
|
|
@ -1 +0,0 @@
|
|||
2746f9ec96f9ce3a345b11f03751136073f7869f
|
||||
|
|
@ -0,0 +1 @@
|
|||
872d97891b63a07475f2ad54b5cd2532ca949b9f
|
||||
|
|
@ -1 +0,0 @@
|
|||
f73773fc39d43df7661609b9f7a733ddfd091af7
|
||||
|
|
@ -0,0 +1 @@
|
|||
ab54d212db5d76d210d632648da23d659b95c23c
|
||||
|
|
@ -1 +0,0 @@
|
|||
be8a20787124cf52c56c5928ef970df2d8a26f51
|
||||
|
|
@ -0,0 +1 @@
|
|||
81ded52ae9e7dfe7a494d1f0c4ae206d245e4f68
|
||||
|
|
@ -1 +0,0 @@
|
|||
8ec1dce97ba5b616e165068225bba873179482e9
|
||||
|
|
@ -0,0 +1 @@
|
|||
09888a8ee3f294b64686e6ce1c838e4b715af3ff
|
||||
|
|
@ -1 +0,0 @@
|
|||
dff1e225fe6bfdf7853663bc48831e9714bf035e
|
||||
|
|
@ -0,0 +1 @@
|
|||
dc991eab909c597c357096ee8a8c1596d6336e4f
|
||||
|
|
@ -1 +0,0 @@
|
|||
ebe2549568386d5c289ec0eb738172f1a0445259
|
||||
|
|
@ -0,0 +1 @@
|
|||
2121b2867debc14ff5263fef0228bcbc00ecfdb3
|
||||
|
|
@ -1 +0,0 @@
|
|||
9d5fdc88f91586bf5d1afa13b9a77302c39b5e7c
|
||||
|
|
@ -0,0 +1 @@
|
|||
c3c4baa42396f44b4be1297e02530115fc77db0c
|
||||
|
|
@ -1 +0,0 @@
|
|||
ab7c7c3c823cb2f8fb1b54fdc82b3e133e8e8344
|
||||
|
|
@ -0,0 +1 @@
|
|||
c9d2393c5587c3948ebfef2bbc864e49a42d64ae
|
||||
|
|
@ -1 +0,0 @@
|
|||
bc34429b8d1a620c58639f376bee9ba425a035d3
|
||||
|
|
@ -0,0 +1 @@
|
|||
76f51eb693a3befa3ff56cca16b111bcaa84a5f9
|
||||
|
|
@ -1 +0,0 @@
|
|||
0bc8d9f00bd34806bc82d01390855ef9dcbea85b
|
||||
|
|
@ -1 +0,0 @@
|
|||
a1e978879d35af3590549437b80679b5c00f27d6
|
||||
|
|
@ -0,0 +1 @@
|
|||
3fbd17d1ba11874cbf6dfb766fd41e4ac1ed1289
|
||||
|
|
@ -1 +0,0 @@
|
|||
1000c919125bb13f265b101341c34bb5af814fd3
|
||||
|
|
@ -0,0 +1 @@
|
|||
bf7aad63c08231f24e9108eb7262cd1950a96158
|
||||
|
|
@ -1 +0,0 @@
|
|||
488e5cfdd4d2d30b161fc45819a82a6984eb0f99
|
||||
|
|
@ -0,0 +1 @@
|
|||
5e24487295d4c927e75d3ea0b7f6724ee9043b80
|
||||
|
|
@ -21,6 +21,11 @@ package org.apache.lucene.search.uhighlight;
|
|||
|
||||
import java.text.BreakIterator;
|
||||
import java.util.Locale;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.io.IOException;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
|
||||
import static org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter.MULTIVAL_SEP_CHAR;
|
||||
|
||||
|
|
@ -76,4 +81,84 @@ class CustomFieldHighlighter extends FieldHighlighter {
|
|||
}
|
||||
return EMPTY_PASSAGE;
|
||||
}
|
||||
|
||||
// TODO: use FieldHighlighter::highlightOffsetsEnums and modify BoundedBreakIteratorScanner to work with it
|
||||
// LUCENE-9093 modified how FieldHighlighter breaks texts into passages,
|
||||
// which doesn't work well with BoundedBreakIteratorScanner
|
||||
// This is the copy of highlightOffsetsEnums before LUCENE-9093.
|
||||
@Override
|
||||
protected Passage[] highlightOffsetsEnums(OffsetsEnum off)
|
||||
throws IOException {
|
||||
|
||||
final int contentLength = this.breakIterator.getText().getEndIndex();
|
||||
|
||||
if (off.nextPosition() == false) {
|
||||
return new Passage[0];
|
||||
}
|
||||
|
||||
PriorityQueue<Passage> passageQueue = new PriorityQueue<>(Math.min(64, maxPassages + 1), (left, right) -> {
|
||||
if (left.getScore() < right.getScore()) {
|
||||
return -1;
|
||||
} else if (left.getScore() > right.getScore()) {
|
||||
return 1;
|
||||
} else {
|
||||
return left.getStartOffset() - right.getStartOffset();
|
||||
}
|
||||
});
|
||||
Passage passage = new Passage(); // the current passage in-progress. Will either get reset or added to queue.
|
||||
|
||||
do {
|
||||
int start = off.startOffset();
|
||||
if (start == -1) {
|
||||
throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
|
||||
}
|
||||
int end = off.endOffset();
|
||||
if (start < contentLength && end > contentLength) {
|
||||
continue;
|
||||
}
|
||||
// See if this term should be part of a new passage.
|
||||
if (start >= passage.getEndOffset()) {
|
||||
passage = maybeAddPassage(passageQueue, passageScorer, passage, contentLength);
|
||||
// if we exceed limit, we are done
|
||||
if (start >= contentLength) {
|
||||
break;
|
||||
}
|
||||
passage.setStartOffset(Math.max(this.breakIterator.preceding(start + 1), 0));
|
||||
passage.setEndOffset(Math.min(this.breakIterator.following(start), contentLength));
|
||||
}
|
||||
// Add this term to the passage.
|
||||
BytesRef term = off.getTerm();// a reference; safe to refer to
|
||||
assert term != null;
|
||||
passage.addMatch(start, end, term, off.freq());
|
||||
} while (off.nextPosition());
|
||||
maybeAddPassage(passageQueue, passageScorer, passage, contentLength);
|
||||
|
||||
Passage[] passages = passageQueue.toArray(new Passage[passageQueue.size()]);
|
||||
// sort in ascending order
|
||||
Arrays.sort(passages, Comparator.comparingInt(Passage::getStartOffset));
|
||||
return passages;
|
||||
}
|
||||
|
||||
// TODO: use FieldHighlighter::maybeAddPassage
|
||||
// After removing CustomFieldHighlighter::highlightOffsetsEnums, remove this method as well.
|
||||
private Passage maybeAddPassage(PriorityQueue<Passage> passageQueue, PassageScorer scorer, Passage passage, int contentLength) {
|
||||
if (passage.getStartOffset() == -1) {
|
||||
// empty passage, we can ignore it
|
||||
return passage;
|
||||
}
|
||||
passage.setScore(scorer.score(passage, contentLength));
|
||||
// new sentence: first add 'passage' to queue
|
||||
if (passageQueue.size() == maxPassages && passage.getScore() < passageQueue.peek().getScore()) {
|
||||
passage.reset(); // can't compete, just reset it
|
||||
} else {
|
||||
passageQueue.offer(passage);
|
||||
if (passageQueue.size() > maxPassages) {
|
||||
passage = passageQueue.poll();
|
||||
passage.reset();
|
||||
} else {
|
||||
passage = new Passage();
|
||||
}
|
||||
}
|
||||
return passage;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
|
|||
public static final Version V_7_5_2 = new Version(7050299, org.apache.lucene.util.Version.LUCENE_8_3_0);
|
||||
public static final Version V_7_5_3 = new Version(7050399, org.apache.lucene.util.Version.LUCENE_8_3_0);
|
||||
public static final Version V_7_6_0 = new Version(7060099, org.apache.lucene.util.Version.LUCENE_8_4_0);
|
||||
public static final Version V_7_7_0 = new Version(7070099, org.apache.lucene.util.Version.LUCENE_8_4_0);
|
||||
public static final Version V_7_7_0 = new Version(7070099, org.apache.lucene.util.Version.LUCENE_8_5_0);
|
||||
public static final Version CURRENT = V_7_7_0;
|
||||
|
||||
private static final ImmutableOpenIntMap<Version> idToVersion;
|
||||
|
|
|
|||
|
|
@ -797,8 +797,8 @@ public abstract class IntervalsSourceProvider implements NamedWriteable, ToXCont
|
|||
BytesRef normalizedTerm = analyzer.normalize(fieldType.name(), term);
|
||||
FuzzyQuery fq = new FuzzyQuery(new Term(fieldType.name(), normalizedTerm),
|
||||
fuzziness.asDistance(term), prefixLength, 128, transpositions);
|
||||
CompiledAutomaton ca = new CompiledAutomaton(fq.toAutomaton());
|
||||
source = XIntervals.multiterm(ca, term);
|
||||
CompiledAutomaton[] automata = fq.getAutomata();
|
||||
source = XIntervals.multiterm(automata[automata.length - 1], term);
|
||||
if (useField != null) {
|
||||
source = Intervals.fixField(useField, source);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -550,7 +550,8 @@ public class IntervalQueryBuilderTests extends AbstractQueryTestCase<IntervalQue
|
|||
|
||||
private static IntervalsSource buildFuzzySource(String term, String label, int prefixLength, boolean transpositions, int editDistance) {
|
||||
FuzzyQuery fq = new FuzzyQuery(new Term("field", term), editDistance, prefixLength, 128, transpositions);
|
||||
return XIntervals.multiterm(new CompiledAutomaton(fq.toAutomaton()), label);
|
||||
CompiledAutomaton[] automata = fq.getAutomata();
|
||||
return XIntervals.multiterm(automata[automata.length - 1], label);
|
||||
}
|
||||
|
||||
public void testFuzzy() throws IOException {
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
2746f9ec96f9ce3a345b11f03751136073f7869f
|
||||
|
|
@ -0,0 +1 @@
|
|||
872d97891b63a07475f2ad54b5cd2532ca949b9f
|
||||
Loading…
Reference in New Issue