Upgrade to Lucene 8.10.1 (#1440)

This commit upgrades to the latest release of lucene 8.10

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
This commit is contained in:
Nick Knize 2021-10-28 10:06:53 -05:00 committed by GitHub
parent 12789f89a3
commit 53334b2ce4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
57 changed files with 50 additions and 44 deletions

View File

@ -1,5 +1,5 @@
opensearch = 2.0.0
lucene = 8.9.0
lucene = 8.10.1
bundled_jdk_vendor = adoptium
bundled_jdk = 17+35

View File

@ -37,6 +37,8 @@ import org.apache.lucene.analysis.en.PorterStemFilterFactory;
import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilterFactory;
import org.apache.lucene.analysis.reverse.ReverseStringFilterFactory;
import org.apache.lucene.analysis.snowball.SnowballPorterFilterFactory;
import org.apache.lucene.analysis.te.TeluguNormalizationFilterFactory;
import org.apache.lucene.analysis.te.TeluguStemFilterFactory;
import org.opensearch.indices.analysis.AnalysisFactoryTestCase;
import java.util.List;
@ -153,6 +155,8 @@ public class CommonAnalysisFactoryTests extends AnalysisFactoryTestCase {
filters.put("brazilianstem", BrazilianStemTokenFilterFactory.class);
filters.put("czechstem", CzechStemTokenFilterFactory.class);
filters.put("germanstem", GermanStemTokenFilterFactory.class);
filters.put("telugunormalization", TeluguNormalizationFilterFactory.class);
filters.put("telugustem", TeluguStemFilterFactory.class);
// this filter is not exposed and should only be used internally
filters.put("fixedshingle", Void.class);
return filters;

View File

@ -0,0 +1 @@
24932a4be7064a99126d80776718845b356abae0

View File

@ -1 +0,0 @@
c52e0f197d8f37fec8e679660a4814b8928e4db2

View File

@ -114,7 +114,7 @@ class ReindexValidator {
return new CharacterRunAutomaton(Automata.makeEmpty());
}
Automaton automaton = Regex.simpleMatchToAutomaton(whitelist.toArray(Strings.EMPTY_ARRAY));
automaton = MinimizationOperations.minimize(automaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
automaton = MinimizationOperations.minimize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
if (Operations.isTotal(automaton)) {
throw new IllegalArgumentException(
"Refusing to start because whitelist "

View File

@ -0,0 +1 @@
a1eec256a25340ba5d432d2800f759db83eb5145

View File

@ -1 +0,0 @@
390ebbb1cdb3ab1e8e5286ae4d77d336fd3895ba

View File

@ -0,0 +1 @@
d9ff6329a9755bbdb7343452bf246e61ae9279d8

View File

@ -1 +0,0 @@
d7553997ad55671bb8c4c023f607f71463da4d0f

View File

@ -0,0 +1 @@
6e78aef6d1b709ed3e27dbc949255e078da08d41

View File

@ -1 +0,0 @@
99ae12d053d27e2b8d78d084e1bf1c893b896554

View File

@ -0,0 +1 @@
c186bf6dd0c2fa6612ba9b0d785ff2d388d32a23

View File

@ -1 +0,0 @@
144cbfb8c5a66ae9ac7abddb8855974599d1ea67

View File

@ -0,0 +1 @@
ebda1884c24bb14ee451b98e7565c86966f8863d

View File

@ -1 +0,0 @@
35f24955ae3d079a262a3aa7ac2ca2449c6d10ee

View File

@ -0,0 +1 @@
2a4bd86c96374cdc5acaf7c0efd5127f2fd3a519

View File

@ -1 +0,0 @@
d2a6fb6327e54c4eccfaa0f8e74953cd69a31e07

View File

@ -0,0 +1 @@
09de2e3fa72355228b2723f958dcb0ec1bc3f31a

View File

@ -1 +0,0 @@
37fd6e23bbb1c146c729bb32b6c043ed0642a73e

View File

@ -0,0 +1 @@
23bb36a98d01100953674c56c20861b29b5a5175

View File

@ -1 +0,0 @@
18f3bbff2b7672ea0b9cc18c8110ef69c763ae6b

View File

@ -0,0 +1 @@
7399c32bc4ba7e37e14a9660ffd7962acf68a802

View File

@ -1 +0,0 @@
fec88b5e71c699ceddc3ae0369481697ac9a5c96

View File

@ -0,0 +1 @@
deb78f6b21d29f964ab267ad59fafb58ef740101

View File

@ -1 +0,0 @@
5c3f72357089f7f0c1ef44bbe7b4c67b6149a5af

View File

@ -0,0 +1 @@
7b91bb886d30c67a8f980d3bdfd6b7826a62d5e7

View File

@ -1 +0,0 @@
9440fdd430b1c2dadbf3bc72656848d61e6f747f

View File

@ -0,0 +1 @@
ec4a2103cb300aab7e6142f1c7778dd505ecb8e2

View File

@ -1 +0,0 @@
94e80bdeab170b0ce1b36a32b6a790d23d7f6d7b

View File

@ -0,0 +1 @@
aa368e9d11660dcfcfaab1a39dd871f05fa2b031

View File

@ -1 +0,0 @@
5ae97803efd3344597f6b6bdf823b18d130e8851

View File

@ -0,0 +1 @@
9de18bf605879647e964fd57ddf3fa6f85ca743e

View File

@ -1 +0,0 @@
09423a6dca2a9ba665e644d86a713d9a6b2b0d3f

View File

@ -0,0 +1 @@
e9cca86ebbe010d375388c5a17216e2d2b2e76bb

View File

@ -1 +0,0 @@
067494d621ba2ef1f2e4da3ef167106f00b52051

View File

@ -0,0 +1 @@
21b70a0996e3408291514d99e3b03800d0bcd657

View File

@ -1 +0,0 @@
c6bda4622abf240da6567a128242f46708fa6c00

View File

@ -0,0 +1 @@
087f52ee3f72f387b802c49a96e4a14b3b05dd21

View File

@ -1 +0,0 @@
95a9d8cf8ca8eaf9f241fd323697d26d211721b2

View File

@ -0,0 +1 @@
82b15ef61297e6d7b0c1f6c37c502d6b77a82f1e

View File

@ -1 +0,0 @@
af9f6c0287465e17a520b93b684474712433b293

View File

@ -0,0 +1 @@
7a3b6eac3e66bb1c6fb05c0cd980e5592adaf96b

View File

@ -1 +0,0 @@
c89f4e78712806e8d5bb4adfb21cf0722ad3f175

View File

@ -0,0 +1 @@
823a5e9d2fd3b5b668d305e0781d0e074e9f2ebb

View File

@ -1 +0,0 @@
0d1238c4e8bf4409b3bb3fbddf2e977b0f19b24b

View File

@ -0,0 +1 @@
92d7e5a178d0df58e0b4d400755ac46bae3eea11

View File

@ -1 +0,0 @@
98cb69950b48b829b6605a003c99aa7eb86fa9eb

View File

@ -74,7 +74,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_1_0_1 = new Version(1000199, org.apache.lucene.util.Version.LUCENE_8_8_2);
public static final Version V_1_1_0 = new Version(1010099, org.apache.lucene.util.Version.LUCENE_8_9_0);
public static final Version V_1_2_0 = new Version(1020099, org.apache.lucene.util.Version.LUCENE_8_9_0);
public static final Version V_2_0_0 = new Version(2000099, org.apache.lucene.util.Version.LUCENE_8_9_0);
public static final Version V_2_0_0 = new Version(2000099, org.apache.lucene.util.Version.LUCENE_8_10_1);
public static final Version CURRENT = V_2_0_0;
public static Version readVersion(StreamInput in) throws IOException {

View File

@ -72,7 +72,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
public static final String NAME = "query_string";
public static final int DEFAULT_MAX_DETERMINED_STATES = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
public static final int DEFAULT_DETERMINIZE_WORK_LIMIT = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
public static final boolean DEFAULT_ENABLE_POSITION_INCREMENTS = true;
public static final boolean DEFAULT_ESCAPE = false;
public static final int DEFAULT_FUZZY_PREFIX_LENGTH = FuzzyQuery.defaultPrefixLength;
@ -160,7 +160,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
private ZoneId timeZone;
/** To limit effort spent determinizing regexp queries. */
private int maxDeterminizedStates = DEFAULT_MAX_DETERMINED_STATES;
private int maxDeterminizedStates = DEFAULT_DETERMINIZE_WORK_LIMIT;
private boolean autoGenerateSynonymsPhraseQuery = true;
@ -658,7 +658,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
String quoteAnalyzer = null;
String queryName = null;
float boost = AbstractQueryBuilder.DEFAULT_BOOST;
int maxDeterminizedStates = QueryStringQueryBuilder.DEFAULT_MAX_DETERMINED_STATES;
int determinizeWorkLimit = QueryStringQueryBuilder.DEFAULT_DETERMINIZE_WORK_LIMIT;
boolean enablePositionIncrements = QueryStringQueryBuilder.DEFAULT_ENABLE_POSITION_INCREMENTS;
boolean escape = QueryStringQueryBuilder.DEFAULT_ESCAPE;
int fuzzyPrefixLength = QueryStringQueryBuilder.DEFAULT_FUZZY_PREFIX_LENGTH;
@ -710,7 +710,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
} else if (ALLOW_LEADING_WILDCARD_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
allowLeadingWildcard = parser.booleanValue();
} else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
maxDeterminizedStates = parser.intValue();
determinizeWorkLimit = parser.intValue();
} else if (ENABLE_POSITION_INCREMENTS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
enablePositionIncrements = parser.booleanValue();
} else if (ESCAPE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
@ -742,7 +742,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
} else if (LENIENT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
lenient = parser.booleanValue();
} else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
maxDeterminizedStates = parser.intValue();
determinizeWorkLimit = parser.intValue();
} else if (TIME_ZONE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
try {
timeZone = parser.text();
@ -784,7 +784,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
queryStringQuery.analyzer(analyzer);
queryStringQuery.quoteAnalyzer(quoteAnalyzer);
queryStringQuery.allowLeadingWildcard(allowLeadingWildcard);
queryStringQuery.maxDeterminizedStates(maxDeterminizedStates);
queryStringQuery.maxDeterminizedStates(determinizeWorkLimit);
queryStringQuery.enablePositionIncrements(enablePositionIncrements);
queryStringQuery.escape(escape);
queryStringQuery.fuzzyPrefixLength(fuzzyPrefixLength);
@ -945,7 +945,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
queryParser.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(this.fuzzyRewrite, LoggingDeprecationHandler.INSTANCE));
queryParser.setMultiTermRewriteMethod(QueryParsers.parseRewriteMethod(this.rewrite, LoggingDeprecationHandler.INSTANCE));
queryParser.setTimeZone(timeZone);
queryParser.setMaxDeterminizedStates(maxDeterminizedStates);
queryParser.setDeterminizeWorkLimit(maxDeterminizedStates);
queryParser.setAutoGenerateMultiTermSynonymsPhraseQuery(autoGenerateSynonymsPhraseQuery);
queryParser.setFuzzyTranspositions(fuzzyTranspositions);

View File

@ -62,7 +62,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
public static final String NAME = "regexp";
public static final int DEFAULT_FLAGS_VALUE = RegexpFlag.ALL.value();
public static final int DEFAULT_MAX_DETERMINIZED_STATES = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
public static final int DEFAULT_DETERMINIZE_WORK_LIMIT = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
public static final boolean DEFAULT_CASE_INSENSITIVITY = false;
private static final ParseField FLAGS_VALUE_FIELD = new ParseField("flags_value");
@ -79,7 +79,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
private int syntaxFlagsValue = DEFAULT_FLAGS_VALUE;
private boolean caseInsensitive = DEFAULT_CASE_INSENSITIVITY;
private int maxDeterminizedStates = DEFAULT_MAX_DETERMINIZED_STATES;
private int maxDeterminizedStates = DEFAULT_DETERMINIZE_WORK_LIMIT;
private String rewrite;
@ -221,7 +221,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
float boost = AbstractQueryBuilder.DEFAULT_BOOST;
int flagsValue = RegexpQueryBuilder.DEFAULT_FLAGS_VALUE;
boolean caseInsensitive = DEFAULT_CASE_INSENSITIVITY;
int maxDeterminizedStates = RegexpQueryBuilder.DEFAULT_MAX_DETERMINIZED_STATES;
int maxDeterminizedStates = RegexpQueryBuilder.DEFAULT_DETERMINIZE_WORK_LIMIT;
String queryName = null;
String currentFieldName = null;
XContentParser.Token token;

View File

@ -782,7 +782,7 @@ public class QueryStringQueryParser extends XQueryParser {
setAnalyzer(forceAnalyzer);
return super.getRegexpQuery(field, termStr);
}
return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getMaxDeterminizedStates(), getMultiTermRewriteMethod(), context);
return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getDeterminizeWorkLimit(), getMultiTermRewriteMethod(), context);
} catch (RuntimeException e) {
if (lenient) {
return newLenientFieldQuery(field, e);

View File

@ -594,9 +594,9 @@ public class IncludeExclude implements Writeable, ToXContentFragment {
a = Automata.makeAnyString();
}
if (exclude != null) {
a = Operations.minus(a, exclude.toAutomaton(), Operations.DEFAULT_MAX_DETERMINIZED_STATES);
a = Operations.minus(a, exclude.toAutomaton(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
} else if (excludeValues != null) {
a = Operations.minus(a, Automata.makeStringUnion(excludeValues), Operations.DEFAULT_MAX_DETERMINIZED_STATES);
a = Operations.minus(a, Automata.makeStringUnion(excludeValues), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
}
return a;
}

View File

@ -226,7 +226,7 @@ public class FuzzyOptions implements ToXContentFragment, Writeable {
private int fuzzyMinLength = FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH;
private int fuzzyPrefixLength = FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX;
private boolean unicodeAware = FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE;
private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
private int maxDeterminizedStates = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
public Builder() {}

View File

@ -156,7 +156,7 @@ public class RegexOptions implements ToXContentFragment, Writeable {
*/
public static class Builder {
private int flagsValue = RegExp.ALL;
private int maxDeterminizedStates = Operations.DEFAULT_MAX_DETERMINIZED_STATES;
private int maxDeterminizedStates = Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
public Builder() {}

View File

@ -720,7 +720,7 @@ public class CompletionFieldMapperTests extends MapperTestCase {
Fuzziness.fromEdits(FuzzyCompletionQuery.DEFAULT_MAX_EDITS),
FuzzyCompletionQuery.DEFAULT_NON_FUZZY_PREFIX,
FuzzyCompletionQuery.DEFAULT_MIN_FUZZY_LENGTH,
Operations.DEFAULT_MAX_DETERMINIZED_STATES,
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
FuzzyCompletionQuery.DEFAULT_TRANSPOSITIONS,
FuzzyCompletionQuery.DEFAULT_UNICODE_AWARE
);
@ -732,7 +732,7 @@ public class CompletionFieldMapperTests extends MapperTestCase {
Mapper fieldMapper = defaultMapper.mappers().getMapper("field");
CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
Query prefixQuery = completionFieldMapper.fieldType()
.regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
.regexpQuery(new BytesRef("co"), RegExp.ALL, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
assertThat(prefixQuery, instanceOf(RegexCompletionQuery.class));
}

View File

@ -754,7 +754,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
() -> queryBuilder.toQuery(createShardContext())
);
assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
assertThat(e.getMessage(), containsString("would result in more than 10000 states"));
assertThat(e.getMessage(), containsString("would require more than 10000 effort"));
}
/**
@ -780,7 +780,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
() -> queryBuilder.toQuery(createShardContext())
);
assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
assertThat(e.getMessage(), containsString("would result in more than 10 states"));
assertThat(e.getMessage(), containsString("would require more than 10 effort"));
}
public void testToQueryFuzzyQueryAutoFuziness() throws Exception {

View File

@ -212,6 +212,8 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
.put("dropifflagged", Void.class)
// LUCENE-9575: recognize arbitrary patterns that include punctuation
.put("patterntyping", Void.class)
.put("telugustem", Void.class)
.put("telugunormalization", Void.class)
.immutableMap();
static final Map<String,Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String,Class<?>>()