Upgrade to Lucene 8.10.1 (#1440)
This commit upgrades to the latest release of lucene 8.10 Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
This commit is contained in:
parent
12789f89a3
commit
53334b2ce4
|
@ -1,5 +1,5 @@
|
|||
opensearch = 2.0.0
|
||||
lucene = 8.9.0
|
||||
lucene = 8.10.1
|
||||
|
||||
bundled_jdk_vendor = adoptium
|
||||
bundled_jdk = 17+35
|
||||
|
|
|
@ -37,6 +37,8 @@ import org.apache.lucene.analysis.en.PorterStemFilterFactory;
|
|||
import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilterFactory;
|
||||
import org.apache.lucene.analysis.reverse.ReverseStringFilterFactory;
|
||||
import org.apache.lucene.analysis.snowball.SnowballPorterFilterFactory;
|
||||
import org.apache.lucene.analysis.te.TeluguNormalizationFilterFactory;
|
||||
import org.apache.lucene.analysis.te.TeluguStemFilterFactory;
|
||||
import org.opensearch.indices.analysis.AnalysisFactoryTestCase;
|
||||
|
||||
import java.util.List;
|
||||
|
@ -153,6 +155,8 @@ public class CommonAnalysisFactoryTests extends AnalysisFactoryTestCase {
|
|||
filters.put("brazilianstem", BrazilianStemTokenFilterFactory.class);
|
||||
filters.put("czechstem", CzechStemTokenFilterFactory.class);
|
||||
filters.put("germanstem", GermanStemTokenFilterFactory.class);
|
||||
filters.put("telugunormalization", TeluguNormalizationFilterFactory.class);
|
||||
filters.put("telugustem", TeluguStemFilterFactory.class);
|
||||
// this filter is not exposed and should only be used internally
|
||||
filters.put("fixedshingle", Void.class);
|
||||
return filters;
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
24932a4be7064a99126d80776718845b356abae0
|
|
@ -1 +0,0 @@
|
|||
c52e0f197d8f37fec8e679660a4814b8928e4db2
|
|
@ -114,7 +114,7 @@ class ReindexValidator {
|
|||
return new CharacterRunAutomaton(Automata.makeEmpty());
|
||||
}
|
||||
Automaton automaton = Regex.simpleMatchToAutomaton(whitelist.toArray(Strings.EMPTY_ARRAY));
|
||||
automaton = MinimizationOperations.minimize(automaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
|
||||
automaton = MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
|
||||
if (Operations.isTotal(automaton)) {
|
||||
throw new IllegalArgumentException(
|
||||
"Refusing to start because whitelist "
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
a1eec256a25340ba5d432d2800f759db83eb5145
|
|
@ -1 +0,0 @@
|
|||
390ebbb1cdb3ab1e8e5286ae4d77d336fd3895ba
|
|
@ -0,0 +1 @@
|
|||
d9ff6329a9755bbdb7343452bf246e61ae9279d8
|
|
@ -1 +0,0 @@
|
|||
d7553997ad55671bb8c4c023f607f71463da4d0f
|
|
@ -0,0 +1 @@
|
|||
6e78aef6d1b709ed3e27dbc949255e078da08d41
|
|
@ -1 +0,0 @@
|
|||
99ae12d053d27e2b8d78d084e1bf1c893b896554
|
|
@ -0,0 +1 @@
|
|||
c186bf6dd0c2fa6612ba9b0d785ff2d388d32a23
|
|
@ -1 +0,0 @@
|
|||
144cbfb8c5a66ae9ac7abddb8855974599d1ea67
|
|
@ -0,0 +1 @@
|
|||
ebda1884c24bb14ee451b98e7565c86966f8863d
|
|
@ -1 +0,0 @@
|
|||
35f24955ae3d079a262a3aa7ac2ca2449c6d10ee
|
|
@ -0,0 +1 @@
|
|||
2a4bd86c96374cdc5acaf7c0efd5127f2fd3a519
|
|
@ -1 +0,0 @@
|
|||
d2a6fb6327e54c4eccfaa0f8e74953cd69a31e07
|
|
@ -0,0 +1 @@
|
|||
09de2e3fa72355228b2723f958dcb0ec1bc3f31a
|
|
@ -1 +0,0 @@
|
|||
37fd6e23bbb1c146c729bb32b6c043ed0642a73e
|
|
@ -0,0 +1 @@
|
|||
23bb36a98d01100953674c56c20861b29b5a5175
|
|
@ -1 +0,0 @@
|
|||
18f3bbff2b7672ea0b9cc18c8110ef69c763ae6b
|
|
@ -0,0 +1 @@
|
|||
7399c32bc4ba7e37e14a9660ffd7962acf68a802
|
|
@ -1 +0,0 @@
|
|||
fec88b5e71c699ceddc3ae0369481697ac9a5c96
|
|
@ -0,0 +1 @@
|
|||
deb78f6b21d29f964ab267ad59fafb58ef740101
|
|
@ -1 +0,0 @@
|
|||
5c3f72357089f7f0c1ef44bbe7b4c67b6149a5af
|
|
@ -0,0 +1 @@
|
|||
7b91bb886d30c67a8f980d3bdfd6b7826a62d5e7
|
|
@ -1 +0,0 @@
|
|||
9440fdd430b1c2dadbf3bc72656848d61e6f747f
|
|
@ -0,0 +1 @@
|
|||
ec4a2103cb300aab7e6142f1c7778dd505ecb8e2
|
|
@ -1 +0,0 @@
|
|||
94e80bdeab170b0ce1b36a32b6a790d23d7f6d7b
|
|
@ -0,0 +1 @@
|
|||
aa368e9d11660dcfcfaab1a39dd871f05fa2b031
|
|
@ -1 +0,0 @@
|
|||
5ae97803efd3344597f6b6bdf823b18d130e8851
|
|
@ -0,0 +1 @@
|
|||
9de18bf605879647e964fd57ddf3fa6f85ca743e
|
|
@ -1 +0,0 @@
|
|||
09423a6dca2a9ba665e644d86a713d9a6b2b0d3f
|
|
@ -0,0 +1 @@
|
|||
e9cca86ebbe010d375388c5a17216e2d2b2e76bb
|
|
@ -1 +0,0 @@
|
|||
067494d621ba2ef1f2e4da3ef167106f00b52051
|
|
@ -0,0 +1 @@
|
|||
21b70a0996e3408291514d99e3b03800d0bcd657
|
|
@ -1 +0,0 @@
|
|||
c6bda4622abf240da6567a128242f46708fa6c00
|
|
@ -0,0 +1 @@
|
|||
087f52ee3f72f387b802c49a96e4a14b3b05dd21
|
|
@ -1 +0,0 @@
|
|||
95a9d8cf8ca8eaf9f241fd323697d26d211721b2
|
|
@ -0,0 +1 @@
|
|||
82b15ef61297e6d7b0c1f6c37c502d6b77a82f1e
|
|
@ -1 +0,0 @@
|
|||
af9f6c0287465e17a520b93b684474712433b293
|
|
@ -0,0 +1 @@
|
|||
7a3b6eac3e66bb1c6fb05c0cd980e5592adaf96b
|
|
@ -1 +0,0 @@
|
|||
c89f4e78712806e8d5bb4adfb21cf0722ad3f175
|
|
@ -0,0 +1 @@
|
|||
823a5e9d2fd3b5b668d305e0781d0e074e9f2ebb
|
|
@ -1 +0,0 @@
|
|||
0d1238c4e8bf4409b3bb3fbddf2e977b0f19b24b
|
|
@ -0,0 +1 @@
|
|||
92d7e5a178d0df58e0b4d400755ac46bae3eea11
|
|
@ -1 +0,0 @@
|
|||
98cb69950b48b829b6605a003c99aa7eb86fa9eb
|
|
@ -74,7 +74,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
|
|||
public static final Version V_1_0_1 = new Version(1000199, org.apache.lucene.util.Version.LUCENE_8_8_2);
|
||||
public static final Version V_1_1_0 = new Version(1010099, org.apache.lucene.util.Version.LUCENE_8_9_0);
|
||||
public static final Version V_1_2_0 = new Version(1020099, org.apache.lucene.util.Version.LUCENE_8_9_0);
|
||||
public static final Version V_2_0_0 = new Version(2000099, org.apache.lucene.util.Version.LUCENE_8_9_0);
|
||||
public static final Version V_2_0_0 = new Version(2000099, org.apache.lucene.util.Version.LUCENE_8_10_1);
|
||||
public static final Version CURRENT = V_2_0_0;
|
||||
|
||||
public static Version readVersion(StreamInput in) throws IOException {
|
||||
|
|
|
@ -72,7 +72,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
|
|||
|
||||
public static final String NAME = "query_string";
|
||||
|
||||
public static final int DEFAULT_MAX_DETERMINED_STATES = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
|
||||
public static final int DEFAULT_DETERMINIZE_WORK_LIMIT = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
|
||||
public static final boolean DEFAULT_ENABLE_POSITION_INCREMENTS = true;
|
||||
public static final boolean DEFAULT_ESCAPE = false;
|
||||
public static final int DEFAULT_FUZZY_PREFIX_LENGTH = FuzzyQuery.defaultPrefixLength;
|
||||
|
@ -160,7 +160,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
|
|||
private ZoneId timeZone;
|
||||
|
||||
/** To limit effort spent determinizing regexp queries. */
|
||||
private int maxDeterminizedStates = DEFAULT_MAX_DETERMINED_STATES;
|
||||
private int maxDeterminizedStates = DEFAULT_DETERMINIZE_WORK_LIMIT;
|
||||
|
||||
private boolean autoGenerateSynonymsPhraseQuery = true;
|
||||
|
||||
|
@ -658,7 +658,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
|
|||
String quoteAnalyzer = null;
|
||||
String queryName = null;
|
||||
float boost = AbstractQueryBuilder.DEFAULT_BOOST;
|
||||
int maxDeterminizedStates = QueryStringQueryBuilder.DEFAULT_MAX_DETERMINED_STATES;
|
||||
int determinizeWorkLimit = QueryStringQueryBuilder.DEFAULT_DETERMINIZE_WORK_LIMIT;
|
||||
boolean enablePositionIncrements = QueryStringQueryBuilder.DEFAULT_ENABLE_POSITION_INCREMENTS;
|
||||
boolean escape = QueryStringQueryBuilder.DEFAULT_ESCAPE;
|
||||
int fuzzyPrefixLength = QueryStringQueryBuilder.DEFAULT_FUZZY_PREFIX_LENGTH;
|
||||
|
@ -710,7 +710,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
|
|||
} else if (ALLOW_LEADING_WILDCARD_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
|
||||
allowLeadingWildcard = parser.booleanValue();
|
||||
} else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
|
||||
maxDeterminizedStates = parser.intValue();
|
||||
determinizeWorkLimit = parser.intValue();
|
||||
} else if (ENABLE_POSITION_INCREMENTS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
|
||||
enablePositionIncrements = parser.booleanValue();
|
||||
} else if (ESCAPE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
|
||||
|
@ -742,7 +742,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
|
|||
} else if (LENIENT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
|
||||
lenient = parser.booleanValue();
|
||||
} else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
|
||||
maxDeterminizedStates = parser.intValue();
|
||||
determinizeWorkLimit = parser.intValue();
|
||||
} else if (TIME_ZONE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
|
||||
try {
|
||||
timeZone = parser.text();
|
||||
|
@ -784,7 +784,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
|
|||
queryStringQuery.analyzer(analyzer);
|
||||
queryStringQuery.quoteAnalyzer(quoteAnalyzer);
|
||||
queryStringQuery.allowLeadingWildcard(allowLeadingWildcard);
|
||||
queryStringQuery.maxDeterminizedStates(maxDeterminizedStates);
|
||||
queryStringQuery.maxDeterminizedStates(determinizeWorkLimit);
|
||||
queryStringQuery.enablePositionIncrements(enablePositionIncrements);
|
||||
queryStringQuery.escape(escape);
|
||||
queryStringQuery.fuzzyPrefixLength(fuzzyPrefixLength);
|
||||
|
@ -945,7 +945,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
|
|||
queryParser.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(this.fuzzyRewrite, LoggingDeprecationHandler.INSTANCE));
|
||||
queryParser.setMultiTermRewriteMethod(QueryParsers.parseRewriteMethod(this.rewrite, LoggingDeprecationHandler.INSTANCE));
|
||||
queryParser.setTimeZone(timeZone);
|
||||
queryParser.setMaxDeterminizedStates(maxDeterminizedStates);
|
||||
queryParser.setDeterminizeWorkLimit(maxDeterminizedStates);
|
||||
queryParser.setAutoGenerateMultiTermSynonymsPhraseQuery(autoGenerateSynonymsPhraseQuery);
|
||||
queryParser.setFuzzyTranspositions(fuzzyTranspositions);
|
||||
|
||||
|
|
|
@ -62,7 +62,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
|
|||
public static final String NAME = "regexp";
|
||||
|
||||
public static final int DEFAULT_FLAGS_VALUE = RegexpFlag.ALL.value();
|
||||
public static final int DEFAULT_MAX_DETERMINIZED_STATES = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
|
||||
public static final int DEFAULT_DETERMINIZE_WORK_LIMIT = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
|
||||
public static final boolean DEFAULT_CASE_INSENSITIVITY = false;
|
||||
|
||||
private static final ParseField FLAGS_VALUE_FIELD = new ParseField("flags_value");
|
||||
|
@ -79,7 +79,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
|
|||
private int syntaxFlagsValue = DEFAULT_FLAGS_VALUE;
|
||||
private boolean caseInsensitive = DEFAULT_CASE_INSENSITIVITY;
|
||||
|
||||
private int maxDeterminizedStates = DEFAULT_MAX_DETERMINIZED_STATES;
|
||||
private int maxDeterminizedStates = DEFAULT_DETERMINIZE_WORK_LIMIT;
|
||||
|
||||
private String rewrite;
|
||||
|
||||
|
@ -221,7 +221,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
|
|||
float boost = AbstractQueryBuilder.DEFAULT_BOOST;
|
||||
int flagsValue = RegexpQueryBuilder.DEFAULT_FLAGS_VALUE;
|
||||
boolean caseInsensitive = DEFAULT_CASE_INSENSITIVITY;
|
||||
int maxDeterminizedStates = RegexpQueryBuilder.DEFAULT_MAX_DETERMINIZED_STATES;
|
||||
int maxDeterminizedStates = RegexpQueryBuilder.DEFAULT_DETERMINIZE_WORK_LIMIT;
|
||||
String queryName = null;
|
||||
String currentFieldName = null;
|
||||
XContentParser.Token token;
|
||||
|
|
|
@ -782,7 +782,7 @@ public class QueryStringQueryParser extends XQueryParser {
|
|||
setAnalyzer(forceAnalyzer);
|
||||
return super.getRegexpQuery(field, termStr);
|
||||
}
|
||||
return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getMaxDeterminizedStates(), getMultiTermRewriteMethod(), context);
|
||||
return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getDeterminizeWorkLimit(), getMultiTermRewriteMethod(), context);
|
||||
} catch (RuntimeException e) {
|
||||
if (lenient) {
|
||||
return newLenientFieldQuery(field, e);
|
||||
|
|
|
@ -594,9 +594,9 @@ public class IncludeExclude implements Writeable, ToXContentFragment {
|
|||
a = Automata.makeAnyString();
|
||||
}
|
||||
if (exclude != null) {
|
||||
a = Operations.minus(a, exclude.toAutomaton(), Operations.DEFAULT_MAX_DETERMINIZED_STATES);
|
||||
a = Operations.minus(a, exclude.toAutomaton(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
|
||||
} else if (excludeValues != null) {
|
||||
a = Operations.minus(a, Automata.makeStringUnion(excludeValues), Operations.DEFAULT_MAX_DETERMINIZED_STATES);
|
||||
a = Operations.minus(a, Automata.makeStringUnion(excludeValues), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
|
|
@ -226,7 +226,7 @@ public class FuzzyOptions implements ToXContentFragment, Writeable {
|
|||
private int fuzzyMinLength = FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH;
|
||||
private int fuzzyPrefixLength = FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX;
|
||||
private boolean unicodeAware = FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE;
|
||||
private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
|
||||
private int maxDeterminizedStates = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
|
||||
|
||||
public Builder() {}
|
||||
|
||||
|
|
|
@ -156,7 +156,7 @@ public class RegexOptions implements ToXContentFragment, Writeable {
|
|||
*/
|
||||
public static class Builder {
|
||||
private int flagsValue = RegExp.ALL;
|
||||
private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
|
||||
private int maxDeterminizedStates = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
|
||||
|
||||
public Builder() {}
|
||||
|
||||
|
|
|
@ -720,7 +720,7 @@ public class CompletionFieldMapperTests extends MapperTestCase {
|
|||
Fuzziness.fromEdits(FuzzyCompletionQuery.DEFAULT_MAX_EDITS),
|
||||
FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX,
|
||||
FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH,
|
||||
Operations.DEFAULT_MAX_DETERMINIZED_STATES,
|
||||
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
|
||||
FuzzyCompletionQuery.DEFAULT_TRANSPOSITIONS,
|
||||
FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE
|
||||
);
|
||||
|
@ -732,7 +732,7 @@ public class CompletionFieldMapperTests extends MapperTestCase {
|
|||
Mapper fieldMapper = defaultMapper.mappers().getMapper("field");
|
||||
CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
|
||||
Query prefixQuery = completionFieldMapper.fieldType()
|
||||
.regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
|
||||
.regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
|
||||
assertThat(prefixQuery, instanceOf(RegexCompletionQuery.class));
|
||||
}
|
||||
|
||||
|
|
|
@ -754,7 +754,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
|
|||
() -> queryBuilder.toQuery(createShardContext())
|
||||
);
|
||||
assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
|
||||
assertThat(e.getMessage(), containsString("would result in more than 10000 states"));
|
||||
assertThat(e.getMessage(), containsString("would require more than 10000 effort"));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -780,7 +780,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
|
|||
() -> queryBuilder.toQuery(createShardContext())
|
||||
);
|
||||
assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
|
||||
assertThat(e.getMessage(), containsString("would result in more than 10 states"));
|
||||
assertThat(e.getMessage(), containsString("would require more than 10 effort"));
|
||||
}
|
||||
|
||||
public void testToQueryFuzzyQueryAutoFuziness() throws Exception {
|
||||
|
|
|
@ -212,6 +212,8 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
|
|||
.put("dropifflagged", Void.class)
|
||||
// LUCENE-9575: recognize arbitrary patterns that include punctuation
|
||||
.put("patterntyping", Void.class)
|
||||
.put("telugustem", Void.class)
|
||||
.put("telugunormalization", Void.class)
|
||||
.immutableMap();
|
||||
|
||||
static final Map<String,Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String,Class<?>>()
|
||||
|
|
Loading…
Reference in New Issue