honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade to lucene-6.4.0-snapshot-ec38570 (#21853) 

Set lucene version to 6.4.0-snapshot-ec38570 and update all the sha1s/license
Fix invalid combo after upgrade in query_string query. split_on_whitespace=false is disallowed if auto_generate_phrase_queries=true
Adapt the expectations of some tests to the new format of the Lucene explain output
This commit is contained in:
Jim Ferenczi 2016-11-29 18:40:31 +01:00 committed by GitHub
parent af1ab68b64
commit d791ddf704
53 changed files with 173 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/version.properties
View File

@ -1,5 +1,5 @@
elasticsearch = 6.0.0-alpha1
lucene = 6.3.0
lucene = 6.4.0-snapshot-ec38570
# optional dependencies
spatial4j = 0.6

1
core/licenses/lucene-analyzers-common-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
494aed699af238c3872a6b65e17939e9cb7ddbe0

1
core/licenses/lucene-analyzers-common-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
770114e0188dd8b4f30e5878b4f6c8677cecf1be

1
core/licenses/lucene-backward-codecs-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
77dede7dff1b833ca2e92d8ab137edb209354d9b

1
core/licenses/lucene-backward-codecs-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
f4eb0257e8419beaa9f84da6a51375fda4e491f2

1
core/licenses/lucene-core-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
d3c87ea89e2f83e401f9cc7f14e4c43945f7f1e1

1
core/licenses/lucene-core-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
c80ad16cd36c41012abb8a8bb1c7328c6d680b4a

1
core/licenses/lucene-grouping-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
2c96d59e318ea66838aeb9c5cfb8b4d27b40953c

1
core/licenses/lucene-grouping-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
070d4e370f4fe0b8a04b2bce5b4381201b0c783f

1
core/licenses/lucene-highlighter-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
4f154d8badfe47fe45503c18fb30f2177f758794

1
core/licenses/lucene-highlighter-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
131d9a86f5943675493a85def0e692842f396458

1
core/licenses/lucene-join-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
79b898117dcfde2981ec6806e420ff218842eca8

1
core/licenses/lucene-join-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
385b2202036b50a764e4d2b032e21496b74a1c8e

1
core/licenses/lucene-memory-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
89edeb404e507d640cb13903acff6953199704a2

1
core/licenses/lucene-memory-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e8742a44ef4849a17d5e59ef36e9a52a8f2370c2

1
core/licenses/lucene-misc-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
02d0e1f5a9df15ac911ad495bad5ea253ab50a9f

1
core/licenses/lucene-misc-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7ce2e4948fb66393a34f4200a6131cfde43e47bd

1
core/licenses/lucene-queries-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
eb7938233c8103223069c7b5b5f785b4d20ddafa

1
core/licenses/lucene-queries-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
6c1c385a597ce797b0049d9b2281b09593e1488a

1
core/licenses/lucene-queryparser-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
e979fb02155cbe81a8d335d6dc41d2ef06be68b6

1
core/licenses/lucene-queryparser-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
fafaa22906c067e6894f9f2b18ad03ded98e2f38

1
core/licenses/lucene-sandbox-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
257387c45c6fa2b77fd6931751f93fdcd798ced4

1
core/licenses/lucene-sandbox-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
19c64a84617f42bb4c11b1e266df4009cd37fdd0

1
core/licenses/lucene-spatial-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
3cf5fe5402b5e34b240b73501c9e97a82428259e

1
core/licenses/lucene-spatial-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
bc8613fb61c0ae95dd3680b0f65e3380c3fd0d6c

1
core/licenses/lucene-spatial-extras-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
1b77ef3740dc885c62d5966fbe9aea1199d344fb

1
core/licenses/lucene-spatial-extras-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
0fa2c3e722294e863f3c70a15e97a18397391fb4

1
core/licenses/lucene-spatial3d-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
aa94b4a8636b3633008640cc5155ad354aebcea5

1
core/licenses/lucene-spatial3d-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
db74c6313965ffdd10d9b19be2eed4ae2c76d2e3

1
core/licenses/lucene-suggest-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
ed5d8ee5cd7edcad5d4ffca2b4540ccc844e9bb0

1
core/licenses/lucene-suggest-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
b85ae1121b5fd56df985615a3cdd7b3879e9b92d

2
core/src/main/java/org/elasticsearch/Version.java
View File

@ -103,7 +103,7 @@ public class Version {
public static final Version V_5_2_0_UNRELEASED = new Version(V_5_2_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001;
public static final Version V_6_0_0_alpha1_UNRELEASED =
new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_0);
public static final Version CURRENT = V_6_0_0_alpha1_UNRELEASED;
// unreleased versions must be added to the above list with the suffix _UNRELEASED (with the exception of CURRENT)

4
core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
View File

@ -937,6 +937,10 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
protected Query doToQuery(QueryShardContext context) throws IOException {
//TODO would be nice to have all the settings in one place: some change though at query execution time
//e.g. field names get expanded to concrete names, defaults get resolved sometimes to settings values etc.
if (splitOnWhitespace == false && autoGeneratePhraseQueries) {
throw new IllegalArgumentException("it is disallowed to disable [split_on_whitespace] " +
"if [auto_generate_phrase_queries] is activated");
}
QueryParserSettings qpSettings;
if (this.escape) {
qpSettings = new QueryParserSettings(org.apache.lucene.queryparser.classic.QueryParser.escape(this.queryString));

4
core/src/main/resources/org/elasticsearch/bootstrap/security.policy
View File

@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" {
//// Very special jar permissions:
//// These are dangerous permissions that we don't want to grant to everything.
grant codeBase "${codebase.lucene-core-6.3.0.jar}" {
grant codeBase "${codebase.lucene-core-6.4.0-snapshot-ec38570.jar}" {
// needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
// java 8 package
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.3.0.jar}" {
permission java.lang.RuntimePermission "accessDeclaredMembers";
};
grant codeBase "${codebase.lucene-misc-6.3.0.jar}" {
grant codeBase "${codebase.lucene-misc-6.4.0-snapshot-ec38570.jar}" {
// needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
permission java.nio.file.LinkPermission "hard";
};

2
core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
View File

@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
grant codeBase "${codebase.lucene-test-framework-6.3.0.jar}" {
grant codeBase "${codebase.lucene-test-framework-6.4.0-snapshot-ec38570.jar}" {
// needed by RamUsageTester
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed for testing hardlinks in StoreRecoveryTests since we install MockFS

13
core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
View File

@ -150,7 +150,10 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
if (randomBoolean()) {
queryStringQueryBuilder.timeZone(randomDateTimeZone().getID());
}
if (queryStringQueryBuilder.autoGeneratePhraseQueries() == false) {
// setSplitOnWhitespace(false) is disallowed when getAutoGeneratePhraseQueries() == true
queryStringQueryBuilder.splitOnWhitespace(randomBoolean());
}
return queryStringQueryBuilder;
}
@ -764,4 +767,14 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
assertThat(e.getMessage(),
containsString("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]"));
}
public void testInvalidCombo() throws IOException {
QueryStringQueryBuilder builder = new QueryStringQueryBuilder("foo bar");
builder.autoGeneratePhraseQueries(true);
builder.splitOnWhitespace(false);
IllegalArgumentException exc =
expectThrows(IllegalArgumentException.class, () -> builder.toQuery(createShardContext()));
assertEquals(exc.getMessage(),
"it is disallowed to disable [split_on_whitespace] if [auto_generate_phrase_queries] is activated");
}
}

34
core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
View File

@ -61,7 +61,6 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
@ -151,8 +150,15 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
assertThat(hit.explanation(), notNullValue());
assertThat(hit.explanation().getDetails().length, equalTo(1));
assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
endsWith("idf(docFreq=100, docCount=100)"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
equalTo("docFreq"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
equalTo(100.0f));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
equalTo("docCount"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
equalTo(100.0f));
assertThat("id[" + hit.id() + "] -> " + hit.explanation().toString(), hit.id(), equalTo(Integer.toString(100 - total - i - 1)));
}
total += hits.length;
@ -179,8 +185,15 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
assertThat(hit.explanation(), notNullValue());
assertThat(hit.explanation().getDetails().length, equalTo(1));
assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
endsWith("idf(docFreq=100, docCount=100)"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
equalTo("docFreq"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
equalTo(100.0f));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
equalTo("docCount"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
equalTo(100.0f));
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(total + i)));
}
total += hits.length;
@ -329,8 +342,15 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
assertThat(hit.explanation(), notNullValue());
assertThat(hit.explanation().getDetails().length, equalTo(1));
assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
endsWith("idf(docFreq=100, docCount=100)"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
equalTo("docFreq"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
equalTo(100.0f));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
equalTo("docCount"));
assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
equalTo(100.0f));
// assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
assertThat("make sure we don't have duplicates", expectedIds.remove(hit.id()), notNullValue());
}

4
docs/reference/query-dsl/query-string-query.asciidoc
View File

@ -85,8 +85,8 @@ for exact matching. Look <<mixing-exact-search-with-stemming,here>> for a
comprehensive example.
|`split_on_whitespace` |Whether query text should be split on whitespace prior to analysis.
Instead the queryparser would parse around only real 'operators'.
Default to `false`.
Instead the queryparser would parse around only real 'operators'. Default to `false`.
It is not allowed to set this option to `false` if `autoGeneratePhraseQueries` is already set to `true`.
|`all_fields` | Perform the query on all fields detected in the mapping that can
be queried. Will be used by default when the `_all` field is disabled and no

96
docs/reference/search/explain.asciidoc
View File

@ -37,44 +37,94 @@ This will yield the following result:
"explanation": {
"value": 1.55077,
"description": "sum of:",
"details" : [ {
"details": [
{
"value": 1.55077,
"description": "weight(message:elasticsearch in 0) [PerFieldSimilarity], result of:",
"details" : [ {
"details": [
{
"value": 1.55077,
"description": "score(doc=0,freq=1.0 = termFreq=1.0\n), product of:",
"details" : [ {
"value" : 1.3862944,
"description" : "idf(docFreq=1, docCount=5)",
"details" : [ ]
}, {
"value" : 1.1186441,
"description" : "tfNorm, computed from:",
"details": [
{ "value" : 1.0, "description" : "termFreq=1.0", "details" : [ ] },
{ "value" : 1.2, "description" : "parameter k1", "details" : [ ] },
{ "value" : 0.75, "description" : "parameter b", "details" : [ ] },
{ "value" : 5.4, "description" : "avgFieldLength", "details" : [ ] },
{ "value" : 4.0, "description" : "fieldLength", "details" : [ ] }
{
"value": 1.3862944,
"description": "idf, computed as log(1 + (docCount - docFreq + 0.5) / (docFreq + 0.5)) from:",
"details": [
{
"value": 1.0,
"description": "docFreq",
"details": []
},
{
"value": 5.0,
"description": "docCount",
"details": []
}
]
} ]
} ]
}, {
},
{
"value": 1.1186441,
"description": "tfNorm, computed as (freq * (k1 + 1)) / (freq + k1 * (1 - b + b * fieldLength / avgFieldLength)) from:",
"details": [
{
"value": 1.0,
"description": "termFreq=1.0",
"details": []
},
{
"value": 1.2,
"description": "parameter k1",
"details": []
},
{
"value": 0.75,
"description": "parameter b",
"details": []
},
{
"value": 5.4,
"description": "avgFieldLength",
"details": []
},
{
"value": 4.0,
"description": "fieldLength",
"details": []
}
]
}
]
}
]
},
{
"value": 0.0,
"description": "match on required clause, product of:",
"details" : [ {
"details": [
{
"value": 0.0,
"description": "# clause",
"details": []
}, {
},
{
"value": 1.0,
"description": "*:*, product of:",
"details": [
{ "value" : 1.0, "description" : "boost", "details" : [ ] },
{ "value" : 1.0, "description" : "queryNorm", "details" : [ ] }
{
"value": 1.0,
"description": "boost",
"details": []
},
{
"value": 1.0,
"description": "queryNorm",
"details": []
}
]
}
]
}
]
} ]
} ]
}
}
--------------------------------------------------

1
modules/lang-expression/licenses/lucene-expressions-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
f9847cdbdd355f9f96993c4c322d6b453f4e84a8

1
modules/lang-expression/licenses/lucene-expressions-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
031d34e0a604a7cbb5c8ba816d49d9f622adaa3f

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
467d808656db028faa3cbc86d386dbf6164a835c

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
0850319baf063c5ee54aecabeaddb95efde8711b

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
bea02277bff7fa0f4d93e6abca94eaf0eec9c84f

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
3a2af1d2e80b9901b3e950f5ac1b6cd1eb408fd3

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
657a1409f539b4a20b5487496a8e4471b33902fd

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7e9243da1482f88a91bd5239316b571259d24341

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
47792194b04e8cd61c3667da50a38adae257b19a

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
2ead714733bb3cc90e9792d76021497946d5af09

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
bcf535520b92821cf04486031214d35d7405571c

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
9a8f3b58e6c672276331f54b5c3be49c8014ec5c

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.3.0.jar.sha1
View File

@ -1 +0,0 @@
82ed82174fae75f93741b8418046bc94e50434f8

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0-snapshot-ec38570.jar.sha1 Normal file
View File

@ -0,0 +1 @@
763b3144b9bc53328e923242a3c6614903ee2d7e