honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable strict duplicate checks for JSON content 

With this commit we enable the Jackson feature 'STRICT_DUPLICATE_DETECTION'
by default. This ensures that JSON keys are always unique. While this has
a performance impact, benchmarking has indicated that the typical drop in
indexing throughput is around 1 - 2%.

As a last resort, we allow users to still disable strict duplicate checks
by setting `-Des.json.strict_duplicate_detection=false` which is
intentionally undocumented.

Closes #19614
This commit is contained in:
Daniel Mitterdorfer 2016-12-14 09:35:53 +01:00 committed by GitHub
parent 49bdd29f91
commit 7e5058037b
19 changed files with 358 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
View File

@ -23,6 +23,7 @@ import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.FastStringReader;
import org.elasticsearch.common.xcontent.XContent;
@ -45,10 +46,31 @@ public class JsonXContent implements XContent {
public static XContentBuilder contentBuilder() throws IOException {
return XContentBuilder.builder(jsonXContent);
}
private static final JsonFactory jsonFactory;
public static final JsonXContent jsonXContent;
/*
* NOTE: This comment is only meant for maintainers of the Elasticsearch code base and is intentionally not a Javadoc comment as it
* describes an undocumented system property.
*
*
* Determines whether the JSON parser will always check for duplicate keys in JSON content. This behavior is enabled by default but
* can be disabled by setting the otherwise undocumented system property "es.json.strict_duplicate_detection" to "false".
*
* Before we've enabled this mode, we had custom duplicate checks in various parts of the code base. As the user can still disable this
* mode and fall back to the legacy duplicate checks, we still need to keep the custom duplicate checks around and we also need to keep
* the tests around.
*
* If this fallback via system property is removed one day in the future you can remove all tests that call this method and also remove
* the corresponding custom duplicate check code.
*
*/
public static boolean isStrictDuplicateDetectionEnabled() {
// Don't allow duplicate keys in JSON content by default but let the user opt out
return Booleans.parseBooleanExact(System.getProperty("es.json.strict_duplicate_detection", "true"));
}
static {
jsonFactory = new JsonFactory();
jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
@ -56,6 +78,7 @@ public class JsonXContent implements XContent {
jsonFactory.configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now...
// Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.core.json.UTF8JsonGenerator#close() method
jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
jsonFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, isStrictDuplicateDetectionEnabled());
jsonXContent = new JsonXContent();
}

2
core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java
View File

@ -67,7 +67,7 @@ public class FieldStatsRequestTests extends ESTestCase {
assertThat(request.getIndexConstraints()[3].getComparison(), equalTo(LTE));
assertThat(request.getIndexConstraints()[4].getField(), equalTo("field5"));
assertThat(request.getIndexConstraints()[4].getValue(), equalTo("2"));
assertThat(request.getIndexConstraints()[4].getProperty(), equalTo(MAX));
assertThat(request.getIndexConstraints()[4].getProperty(), equalTo(MIN));
assertThat(request.getIndexConstraints()[4].getComparison(), equalTo(GT));
assertThat(request.getIndexConstraints()[5].getField(), equalTo("field5"));
assertThat(request.getIndexConstraints()[5].getValue(), equalTo("9"));

3
core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.common.settings.loader;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.ESTestCase;
import static org.hamcrest.CoreMatchers.containsString;
@ -48,6 +49,8 @@ public class JsonSettingsLoaderTests extends ESTestCase {
}
public void testDuplicateKeysThrowsException() {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
final String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}";
final SettingsException e = expectThrows(SettingsException.class, () -> Settings.builder().loadFromSource(json).build());
assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);

2
core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
View File

@ -169,6 +169,8 @@ public class ConstructingObjectParserTests extends ESTestCase {
}
public void testRepeatedConstructorParam() throws IOException {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
XContentParser parser = createParser(JsonXContent.jsonXContent,
"{\n"
+ " \"vegetable\": 1,\n"

10
core/src/test/java/org/elasticsearch/common/xcontent/json/JsonXContentTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.common.xcontent.json;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParseException;
import org.elasticsearch.common.xcontent.BaseXContentTestCase;
import org.elasticsearch.common.xcontent.XContentType;
@ -39,4 +40,13 @@ public class JsonXContentTests extends BaseXContentTestCase {
JsonGenerator generator = new JsonFactory().createGenerator(os);
doTestBigInteger(generator, os);
}
public void testChecksForDuplicates() throws Exception {
assumeTrue("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
JsonParseException pex = expectThrows(JsonParseException.class,
() -> XContentType.JSON.xContent().createParser("{ \"key\": 1, \"key\": 2 }").map());
assertEquals("Duplicate field 'key'", pex.getMessage());
}
}

20
core/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
View File

@ -83,8 +83,8 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
assertThat(copyTestMap.get("type").toString(), is("text"));
List<String> copyToList = (List<String>) copyTestMap.get("copy_to");
assertThat(copyToList.size(), equalTo(2));
assertThat(copyToList.get(0).toString(), equalTo("another_field"));
assertThat(copyToList.get(1).toString(), equalTo("cyclic_test"));
assertThat(copyToList.get(0), equalTo("another_field"));
assertThat(copyToList.get(1), equalTo("cyclic_test"));
// Check data parsing
BytesReference json = jsonBuilder().startObject()
@ -312,7 +312,6 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
public void testCopyToNestedField() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
for (boolean mapped : new boolean[] {true, false}) {
XContentBuilder mapping = jsonBuilder().startObject()
.startObject("type")
.startObject("properties")
@ -342,14 +341,14 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
.value("n1.target") // should go to the parent doc
.value("n1.n2.target") // should go to the current doc
.endArray()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject();
for (int i = 0; i < 3; ++i) {
if (mapped) {
mapping = mapping.startObject("target").field("type", "long").field("doc_values", false).endObject();
}
mapping = mapping.endObject().endObject();
}
mapping = mapping.endObject();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
@ -409,7 +408,6 @@ public class CopyToMapperTests extends ESSingleNodeTestCase {
assertFieldValue(root, "n1.target");
assertFieldValue(root, "n1.n2.target");
}
}
public void testCopyToDynamicNestedObjectParsing() throws Exception {
String mapping = jsonBuilder().startObject().startObject("type1")

3
core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.test.AbstractQueryTestCase;
import org.hamcrest.Matchers;
@ -339,6 +340,8 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
* test that two queries in object throws error
*/
public void testTooManyQueriesInObject() throws IOException {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
String clauseType = randomFrom("must", "should", "must_not", "filter");
// should also throw error if invalid query is preceded by a valid one
String query = "{\n" +

3
core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.index.query;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.test.AbstractQueryTestCase;
@ -65,6 +66,8 @@ public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase<Consta
* test that multiple "filter" elements causes {@link ParsingException}
*/
public void testMultipleFilterElements() throws IOException {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
String queryString = "{ \"" + ConstantScoreQueryBuilder.NAME + "\" : {\n" +
"\"filter\" : { \"term\": { \"foo\": \"a\" } },\n" +
"\"filter\" : { \"term\": { \"foo\": \"x\" } },\n" +

3
core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
View File

@ -38,6 +38,7 @@ import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.AbstractQueryBuilder;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
@ -730,6 +731,8 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
}
public void testMalformedQueryMultipleQueryElements() throws IOException {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
String json = "{\n" +
" \"function_score\":{\n" +
" \"query\":{\n" +

15
core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
View File

@ -21,23 +21,18 @@ package org.elasticsearch.search.aggregations;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.env.Environment;
import org.elasticsearch.test.AbstractQueryTestCase;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.test.AbstractQueryTestCase;
import org.elasticsearch.test.ESTestCase;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -68,11 +63,7 @@ public class AggregatorParsingTests extends ESTestCase {
Settings settings = Settings.builder().put("node.name", AbstractQueryTestCase.class.toString())
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false).build();
IndicesModule indicesModule = new IndicesModule(Collections.emptyList()) ;
SearchModule searchModule = new SearchModule(settings, false, emptyList());
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.addAll(indicesModule.getNamedWriteables());
entries.addAll(searchModule.getNamedWriteables());
aggParsers = searchModule.getSearchRequestParsers().aggParsers;
// create some random type with some default field, those types will
// stick around for all of the subclasses
@ -113,6 +104,8 @@ public class AggregatorParsingTests extends ESTestCase {
}
public void testTwoAggs() throws Exception {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
XContentBuilder source = JsonXContent.contentBuilder()
.startObject()
.startObject("by_date")
@ -187,6 +180,8 @@ public class AggregatorParsingTests extends ESTestCase {
}
public void testSameAggregationName() throws Exception {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
final String name = randomAsciiOfLengthBetween(1, 10);
XContentBuilder source = JsonXContent.contentBuilder()
.startObject()

84
core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
View File

@ -506,16 +506,25 @@ public class SimpleNestedIT extends ESIntegTestCase {
public void testSortNestedWithNestedFilter() throws Exception {
assertAcked(prepareCreate("test")
.addMapping("type1", XContentFactory.jsonBuilder().startObject()
.addMapping("type1", XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("grand_parent_values").field("type", "long").endObject()
.startObject("parent").field("type", "nested")
.startObject("grand_parent_values")
.field("type", "long")
.endObject()
.startObject("parent")
.field("type", "nested")
.startObject("properties")
.startObject("parent_values").field("type", "long").endObject()
.startObject("child").field("type", "nested")
.startObject("parent_values")
.field("type", "long")
.endObject()
.startObject("child")
.field("type", "nested")
.startObject("properties")
.startObject("child_values").field("type", "long").endObject()
.startObject("child_values")
.field("type", "long")
.endObject()
.endObject()
.endObject()
.endObject()
@ -526,99 +535,120 @@ public class SimpleNestedIT extends ESIntegTestCase {
ensureGreen();
// sum: 11
client().prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder().startObject()
client().prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder()
.startObject()
.field("grand_parent_values", 1L)
.startObject("parent")
.startArray("parent")
.startObject()
.field("filter", false)
.field("parent_values", 1L)
.startObject("child")
.startArray("child")
.startObject()
.field("filter", true)
.field("child_values", 1L)
.startObject("child_obj")
.field("value", 1L)
.endObject()
.endObject()
.startObject("child")
.startObject()
.field("filter", false)
.field("child_values", 6L)
.endObject()
.endArray()
.endObject()
.startObject("parent")
.startObject()
.field("filter", true)
.field("parent_values", 2L)
.startObject("child")
.startArray("child")
.startObject()
.field("filter", false)
.field("child_values", -1L)
.endObject()
.startObject("child")
.startObject()
.field("filter", false)
.field("child_values", 5L)
.endObject()
.endArray()
.endObject()
.endArray()
.endObject()).execute().actionGet();
// sum: 7
client().prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject()
client().prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder()
.startObject()
.field("grand_parent_values", 2L)
.startObject("parent")
.startArray("parent")
.startObject()
.field("filter", false)
.field("parent_values", 2L)
.startObject("child")
.startArray("child")
.startObject()
.field("filter", true)
.field("child_values", 2L)
.startObject("child_obj")
.field("value", 2L)
.endObject()
.endObject()
.startObject("child")
.startObject()
.field("filter", false)
.field("child_values", 4L)
.endObject()
.endArray()
.endObject()
.startObject("parent")
.startObject()
.field("parent_values", 3L)
.field("filter", true)
.startObject("child")
.startArray("child")
.startObject()
.field("child_values", -2L)
.field("filter", false)
.endObject()
.startObject("child")
.startObject()
.field("filter", false)
.field("child_values", 3L)
.endObject()
.endArray()
.endObject()
.endArray()
.endObject()).execute().actionGet();
// sum: 2
client().prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject()
client().prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder()
.startObject()
.field("grand_parent_values", 3L)
.startObject("parent")
.startArray("parent")
.startObject()
.field("parent_values", 3L)
.field("filter", false)
.startObject("child")
.startArray("child")
.startObject()
.field("filter", true)
.field("child_values", 3L)
.startObject("child_obj")
.field("value", 3L)
.endObject()
.endObject()
.startObject("child")
.startObject()
.field("filter", false)
.field("child_values", 1L)
.endObject()
.endArray()
.endObject()
.startObject("parent")
.startObject()
.field("parent_values", 4L)
.field("filter", true)
.startObject("child")
.startArray("child")
.startObject()
.field("filter", false)
.field("child_values", -3L)
.endObject()
.startObject("child")
.startObject()
.field("filter", false)
.field("child_values", 1L)
.endObject()
.endArray()
.endObject()
.endArray()
.endObject()).execute().actionGet();
refresh();

4
core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
View File

@ -149,9 +149,13 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
"Required [field]");
// test two fieldnames
if (JsonXContent.isStrictDuplicateDetectionEnabled()) {
logger.info("Skipping test as it uses a custom duplicate check that is obsolete when strict duplicate checks are enabled.");
} else {
directGenerator = "{ \"field\" : \"f1\", \"field\" : \"f2\" }";
assertIllegalXContent(directGenerator, ParsingException.class,
"[direct_generator] failed to parse field [field]");
}
// test unknown field
directGenerator = "{ \"unknown_param\" : \"f1\" }";

2
core/src/test/resources/org/elasticsearch/action/fieldstats/fieldstats-index-constraints-request.json
View File

@ -22,7 +22,7 @@
}
},
"field5": {
"max_value" : {
"min_value" : {
"gt": 2
},
"max_value" : {

5
docs/reference/migration/migrate_6_0/rest.asciidoc
View File

@ -8,6 +8,11 @@ This feature was removed in the 5.x series, but a backwards-compatibility layer
system property `elasticsearch.json.allow_unquoted_field_names`. This backwards-compatibility layer
has been removed in Elasticsearch 6.0.0.
==== Duplicate Keys in JSON
In previous versions of Elasticsearch, JSON documents were allowed to contain duplicate keys. Elasticsearch 6.0.0
enforces that all keys are unique.
==== Analyze API changes
The deprecated request parameters and plain text in request body has been removed. Define parameters in request body.

20
modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
View File

@ -196,10 +196,24 @@ public class MoreExpressionTests extends ESIntegTestCase {
public void testMultiValueMethods() throws Exception {
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double0", "type=double", "double1", "type=double", "double2", "type=double"));
ensureGreen("test");
Map<String, Object> doc1 = new HashMap<>();
doc1.put("double0", new Double[]{5.0d, 1.0d, 1.5d});
doc1.put("double1", new Double[]{1.2d, 2.4d});
doc1.put("double2", 3.0d);
Map<String, Object> doc2 = new HashMap<>();
doc2.put("double0", 5.0d);
doc2.put("double1", 3.0d);
Map<String, Object> doc3 = new HashMap<>();
doc3.put("double0", new Double[]{5.0d, 1.0d, 1.5d, -1.5d});
doc3.put("double1", 4.0d);
indexRandom(true,
client().prepareIndex("test", "doc", "1").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double1", "1.2", "double1", "2.4", "double2", "3.0"),
client().prepareIndex("test", "doc", "2").setSource("double0", "5.0", "double1", "3.0"),
client().prepareIndex("test", "doc", "3").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double0", "-1.5", "double1", "4.0"));
client().prepareIndex("test", "doc", "1").setSource(doc1),
client().prepareIndex("test", "doc", "2").setSource(doc2),
client().prepareIndex("test", "doc", "3").setSource(doc3));
SearchResponse rsp = buildRequest("doc['double0'].count() + doc['double1'].count()").get();

3
modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
View File

@ -184,6 +184,9 @@ public class TransportPercolateAction extends HandledTransportAction<PercolateRe
}
} else if (token.isValue()) {
if ("size".equals(currentFieldName)) {
if (percolateRequest.onlyCount()) {
throw new IllegalArgumentException("Cannot set size if onlyCount == true");
}
searchSource.field("size", parser.intValue());
} else if ("sort".equals(currentFieldName)) {
searchSource.field("sort", parser.text());

45
modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
View File

@ -94,8 +94,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
.setIndices(INDEX_NAME)
.setDocumentType("type")
.setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()))
.setSize(expectedCount[i % numUniqueQueries]);
.setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()));
SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values());
percolateRequestBuilder.addAggregation(AggregationBuilders.terms("a").field("field2").collectMode(aggCollectionMode));
@ -103,15 +102,17 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
if (randomBoolean()) {
percolateRequestBuilder.setPercolateQuery(matchAllQuery());
}
if (randomBoolean()) {
percolateRequestBuilder.setScore(true);
} else {
percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
}
boolean countOnly = randomBoolean();
if (countOnly) {
percolateRequestBuilder.setOnlyCount(countOnly);
} else {
// can only set size if we also keep track of matches (i.e. countOnly == false)
if (randomBoolean()) {
percolateRequestBuilder.setScore(true).setSize(expectedCount[i % numUniqueQueries]);
} else {
percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
}
}
PercolateResponse response = percolateRequestBuilder.execute().actionGet();
@ -161,8 +162,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
.setIndices(INDEX_NAME)
.setDocumentType("type")
.setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()))
.setSize(expectedCount[i % numUniqueQueries]);
.setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()));
SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values());
percolateRequestBuilder.addAggregation(AggregationBuilders.terms("a").field("field2").collectMode(aggCollectionMode));
@ -170,15 +170,17 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
if (randomBoolean()) {
percolateRequestBuilder.setPercolateQuery(matchAllQuery());
}
if (randomBoolean()) {
percolateRequestBuilder.setScore(true);
} else {
percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
}
boolean countOnly = randomBoolean();
if (countOnly) {
percolateRequestBuilder.setOnlyCount(countOnly);
} else {
// can only set size if we also keep track of matches (i.e. countOnly == false)
if (randomBoolean()) {
percolateRequestBuilder.setScore(true).setSize(expectedCount[i % numUniqueQueries]);
} else {
percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
}
}
percolateRequestBuilder.addAggregation(PipelineAggregatorBuilders.maxBucket("max_a", "a>_count"));
@ -243,8 +245,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
.setIndices(INDEX_NAME)
.setDocumentType("type")
.setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()))
.setSize(numQueries);
.setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()));
SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values());
percolateRequestBuilder.addAggregation(AggregationBuilders.terms("terms").field("field2").collectMode(aggCollectionMode)
@ -253,15 +254,17 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
if (randomBoolean()) {
percolateRequestBuilder.setPercolateQuery(matchAllQuery());
}
if (randomBoolean()) {
percolateRequestBuilder.setScore(true);
} else {
percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
}
boolean countOnly = randomBoolean();
if (countOnly) {
percolateRequestBuilder.setOnlyCount(countOnly);
} else {
// can only set size if we also keep track of matches (i.e. countOnly == false)
if (randomBoolean()) {
percolateRequestBuilder.setScore(true).setSize(numQueries);
} else {
percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
}
}
percolateRequestBuilder.addAggregation(PipelineAggregatorBuilders.maxBucket("max_terms", "terms>_count"));

52
modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
View File

@ -1031,24 +1031,28 @@ public class PercolatorIT extends ESIntegTestCase {
refresh();
boolean onlyCount = randomBoolean();
PercolateResponse response = preparePercolate(client())
PercolateRequestBuilder builder = preparePercolate(client())
.setIndices(INDEX_NAME).setDocumentType("my-type")
.setOnlyCount(onlyCount)
.setPercolateDoc(docBuilder().setDoc("field", "value"))
.setSize((int) totalQueries)
.execute().actionGet();
.setPercolateDoc(docBuilder().setDoc("field", "value"));
if (!onlyCount) {
builder.setSize((int) totalQueries);
}
PercolateResponse response = builder.execute().actionGet();
assertMatchCount(response, totalQueries);
if (!onlyCount) {
assertThat(response.getMatches().length, equalTo((int) totalQueries));
}
int size = randomIntBetween(0, (int) totalQueries - 1);
response = preparePercolate(client())
builder = preparePercolate(client())
.setIndices(INDEX_NAME).setDocumentType("my-type")
.setOnlyCount(onlyCount)
.setPercolateDoc(docBuilder().setDoc("field", "value"))
.setSize(size)
.execute().actionGet();
.setPercolateDoc(docBuilder().setDoc("field", "value"));
if (!onlyCount) {
builder.setSize(size);
}
response = builder.execute().actionGet();
assertMatchCount(response, totalQueries);
if (!onlyCount) {
assertThat(response.getMatches().length, equalTo(size));
@ -1060,13 +1064,15 @@ public class PercolatorIT extends ESIntegTestCase {
int runs = randomIntBetween(3, 16);
for (int i = 0; i < runs; i++) {
onlyCount = randomBoolean();
response = preparePercolate(client())
builder = preparePercolate(client())
.setIndices(INDEX_NAME).setDocumentType("my-type")
.setOnlyCount(onlyCount)
.setPercolateDoc(docBuilder().setDoc("field", "value"))
.setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)))
.setSize((int) numQueriesPerLevel)
.execute().actionGet();
.setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)));
if (!onlyCount) {
builder.setSize((int) numQueriesPerLevel);
}
response = builder.execute().actionGet();
assertMatchCount(response, numQueriesPerLevel);
if (!onlyCount) {
assertThat(response.getMatches().length, equalTo((int) numQueriesPerLevel));
@ -1075,13 +1081,15 @@ public class PercolatorIT extends ESIntegTestCase {
for (int i = 0; i < runs; i++) {
onlyCount = randomBoolean();
response = preparePercolate(client())
builder = preparePercolate(client())
.setIndices(INDEX_NAME).setDocumentType("my-type")
.setOnlyCount(onlyCount)
.setPercolateDoc(docBuilder().setDoc("field", "value"))
.setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)))
.setSize((int) numQueriesPerLevel)
.execute().actionGet();
.setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)));
if (!onlyCount) {
builder.setSize((int) numQueriesPerLevel);
}
response = builder.execute().actionGet();
assertMatchCount(response, numQueriesPerLevel);
if (!onlyCount) {
assertThat(response.getMatches().length, equalTo((int) numQueriesPerLevel));
@ -1091,13 +1099,15 @@ public class PercolatorIT extends ESIntegTestCase {
for (int i = 0; i < runs; i++) {
onlyCount = randomBoolean();
size = randomIntBetween(0, (int) numQueriesPerLevel - 1);
response = preparePercolate(client())
builder = preparePercolate(client())
.setIndices(INDEX_NAME).setDocumentType("my-type")
.setOnlyCount(onlyCount)
.setSize(size)
.setPercolateDoc(docBuilder().setDoc("field", "value"))
.setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)))
.execute().actionGet();
.setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)));
if (!onlyCount) {
builder.setSize(size);
}
response = builder.execute().actionGet();
assertMatchCount(response, numQueriesPerLevel);
if (!onlyCount) {
assertThat(response.getMatches().length, equalTo(size));
@ -1726,7 +1736,7 @@ public class PercolatorIT extends ESIntegTestCase {
.setPercolateDoc(docBuilder().setDoc(doc))
.get();
assertMatchCount(response, 3L);
response = preparePercolate(client()).setScore(randomBoolean()).setSortByScore(randomBoolean()).setOnlyCount(randomBoolean()).setSize(10).setPercolateQuery(QueryBuilders.termQuery("text", "foo"))
response = preparePercolate(client()).setScore(randomBoolean()).setSortByScore(randomBoolean()).setOnlyCount(randomBoolean()).setPercolateQuery(QueryBuilders.termQuery("text", "foo"))
.setIndices(INDEX_NAME).setDocumentType("doc")
.setPercolateDoc(docBuilder().setDoc(doc))
.get();

5
test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.rest.yaml.restspec;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.yaml.YamlXContent;
import org.elasticsearch.test.ESTestCase;
@ -71,6 +72,8 @@ public class ClientYamlSuiteRestApiParserFailingTests extends ESTestCase {
}
public void testDuplicateParts() throws Exception {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
parseAndExpectFailure("{\n" +
" \"ping\": {" +
" \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
@ -103,6 +106,8 @@ public class ClientYamlSuiteRestApiParserFailingTests extends ESTestCase {
}
public void testDuplicateParams() throws Exception {
assumeFalse("Test only makes sense if JSON parser doesn't have strict duplicate checks enabled",
JsonXContent.isStrictDuplicateDetectionEnabled());
parseAndExpectFailure("{\n" +
" \"ping\": {" +
" \"documentation\": \"http://www.elasticsearch.org/guide/\"," +