From c85ac402b01ec06b3f41e8245ed5acd59e95a1e6 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 27 Jun 2017 17:44:10 +0200 Subject: [PATCH] test: Make many percolator integration tests real integration tests --- .../percolator/PercolatorQuerySearchIT.java | 201 ++--------------- .../PercolatorQuerySearchTests.java | 205 ++++++++++++++++++ 2 files changed, 222 insertions(+), 184 deletions(-) create mode 100644 modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 9bbf0a284b1..cbd56400cab 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -23,36 +23,19 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.query.MultiMatchQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.MockScriptPlugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESSingleNodeTestCase; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; +import org.elasticsearch.test.ESIntegTestCase; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder; @@ -67,6 +50,7 @@ import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery; import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery; import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; import static org.hamcrest.Matchers.containsString; @@ -75,44 +59,10 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.core.IsNull.notNullValue; -public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { - - @Override - protected Collection> getPlugins() { - return Arrays.asList(PercolatorPlugin.class, CustomScriptPlugin.class); - } - - public static class CustomScriptPlugin extends MockScriptPlugin { - @Override - protected Map, Object>> pluginScripts() { - Map, Object>> scripts = new HashMap<>(); - scripts.put("1==1", vars -> Boolean.TRUE); - scripts.put("use_fielddata_please", vars -> { - LeafDocLookup leafDocLookup = (LeafDocLookup) vars.get("_doc"); - ScriptDocValues scriptDocValues = leafDocLookup.get("employees.name"); - return "virginia_potts".equals(scriptDocValues.get(0)); - }); - return scripts; - } - } - - public void testPercolateScriptQuery() throws IOException { - client().admin().indices().prepareCreate("index").addMapping("type", "query", "type=percolator").get(); - client().prepareIndex("index", "type", "1") - .setSource(jsonBuilder().startObject().field("query", QueryBuilders.scriptQuery( - new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1==1", Collections.emptyMap()))).endObject()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .execute().actionGet(); - SearchResponse response = client().prepareSearch("index") - .setQuery(new PercolateQueryBuilder("query", jsonBuilder().startObject().field("field1", "b").endObject().bytes(), - XContentType.JSON)) - .get(); - assertHitCount(response, 1); - assertSearchHits(response, "1"); - } +public class PercolatorQuerySearchIT extends ESIntegTestCase { public void testPercolatorQuery() throws Exception { - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") ); @@ -160,7 +110,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { } public void testPercolatorRangeQueries() throws Exception { - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=long", "field2", "type=double", "field3", "type=ip", "field4", "type=date", "query", "type=percolator") ); @@ -269,7 +219,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { } public void testPercolatorQueryExistingDocument() throws Exception { - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") ); @@ -318,7 +268,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { } public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception { - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "_source", "enabled=false", "field1", "type=keyword", "query", "type=percolator") ); @@ -340,7 +290,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { } public void testPercolatorSpecificQueries() throws Exception { - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=text", "field2", "type=text", "query", "type=percolator") ); @@ -418,7 +368,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { } else if (randomBoolean()) { fieldMapping.append(",index_options=offsets"); } - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field1", fieldMapping, "query", "type=percolator") ); client().prepareIndex("test", "type", "1") @@ -461,7 +411,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { } public void testTakePositionOffsetGapIntoAccount() throws Exception { - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field", "type=text,position_increment_gap=5", "query", "type=percolator") ); client().prepareIndex("test", "type", "1") @@ -484,13 +434,13 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { public void testManyPercolatorFields() throws Exception { String queryFieldName = randomAlphaOfLength(8); - createIndex("test1", client().admin().indices().prepareCreate("test1") + assertAcked(client().admin().indices().prepareCreate("test1") .addMapping("type", queryFieldName, "type=percolator", "field", "type=keyword") ); - createIndex("test2", client().admin().indices().prepareCreate("test2") + assertAcked(client().admin().indices().prepareCreate("test2") .addMapping("type", queryFieldName, "type=percolator", "second_query_field", "type=percolator", "field", "type=keyword") ); - createIndex("test3", client().admin().indices().prepareCreate("test3") + assertAcked(client().admin().indices().prepareCreate("test3") .addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("field") .field("type", "keyword") @@ -510,9 +460,9 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { public void testWithMultiplePercolatorFields() throws Exception { String queryFieldName = randomAlphaOfLength(8); - createIndex("test1", client().admin().indices().prepareCreate("test1") + assertAcked(client().admin().indices().prepareCreate("test1") .addMapping("type", queryFieldName, "type=percolator", "field", "type=keyword")); - createIndex("test2", client().admin().indices().prepareCreate("test2") + assertAcked(client().admin().indices().prepareCreate("test2") .addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("field") .field("type", "keyword") @@ -578,7 +528,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { .startObject("companyname").field("type", "text").endObject().startObject("employee").field("type", "nested") .startObject("properties").startObject("name").field("type", "text").endObject().endObject().endObject().endObject() .endObject(); - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("employee", mapping) ); client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject() @@ -628,125 +578,8 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase { assertHitCount(response, 0); } - public void testPercolateQueryWithNestedDocuments_doNotLeakBitsetCacheEntries() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder(); - mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject() - .startObject("query").field("type", "percolator").endObject() - .startObject("employee").field("type", "nested").startObject("properties") - .startObject("name").field("type", "text").endObject().endObject().endObject().endObject() - .endObject(); - createIndex("test", client().admin().indices().prepareCreate("test") - // to avoid normal document from being cached by BitsetFilterCache - .setSettings(Settings.builder().put(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING.getKey(), false)) - .addMapping("employee", mapping) - ); - client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject() - .field("query", QueryBuilders.nestedQuery("employee", - QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg) - ).endObject()) - .get(); - client().admin().indices().prepareRefresh().get(); - - for (int i = 0; i < 32; i++) { - SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", - XContentFactory.jsonBuilder() - .startObject().field("companyname", "stark") - .startArray("employee") - .startObject().field("name", "virginia potts").endObject() - .startObject().field("name", "tony stark").endObject() - .endArray() - .endObject().bytes(), XContentType.JSON)) - .addSort("_doc", SortOrder.ASC) - // size 0, because other wise load bitsets for normal document in FetchPhase#findRootDocumentIfNested(...) - .setSize(0) - .get(); - assertHitCount(response, 1); - } - - // We can't check via api... because BitsetCacheListener requires that it can extract shardId from index reader - // and for percolator it can't do that, but that means we don't keep track of - // memory for BitsetCache in case of percolator - long bitsetSize = client().admin().cluster().prepareClusterStats().get() - .getIndicesStats().getSegments().getBitsetMemoryInBytes(); - assertEquals("The percolator works with in-memory index and therefor shouldn't use bitset cache", 0L, bitsetSize); - } - - public void testPercolateQueryWithNestedDocuments_doLeakFieldDataCacheEntries() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder(); - mapping.startObject(); - { - mapping.startObject("properties"); - { - mapping.startObject("query"); - mapping.field("type", "percolator"); - mapping.endObject(); - } - { - mapping.startObject("companyname"); - mapping.field("type", "text"); - mapping.endObject(); - } - { - mapping.startObject("employees"); - mapping.field("type", "nested"); - { - mapping.startObject("properties"); - { - mapping.startObject("name"); - mapping.field("type", "text"); - mapping.field("fielddata", true); - mapping.endObject(); - } - mapping.endObject(); - } - mapping.endObject(); - } - mapping.endObject(); - } - mapping.endObject(); - createIndex("test", client().admin().indices().prepareCreate("test") - .addMapping("employee", mapping) - ); - Script script = new Script(ScriptType.INLINE, MockScriptPlugin.NAME, "use_fielddata_please", Collections.emptyMap()); - client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject() - .field("query", QueryBuilders.nestedQuery("employees", - QueryBuilders.scriptQuery(script), ScoreMode.Avg) - ).endObject()).get(); - client().admin().indices().prepareRefresh().get(); - XContentBuilder doc = jsonBuilder(); - doc.startObject(); - { - doc.field("companyname", "stark"); - doc.startArray("employees"); - { - doc.startObject(); - doc.field("name", "virginia_potts"); - doc.endObject(); - } - { - doc.startObject(); - doc.field("name", "tony_stark"); - doc.endObject(); - } - doc.endArray(); - } - doc.endObject(); - for (int i = 0; i < 32; i++) { - SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", doc.bytes(), XContentType.JSON)) - .addSort("_doc", SortOrder.ASC) - .get(); - assertHitCount(response, 1); - } - - long fieldDataSize = client().admin().cluster().prepareClusterStats().get() - .getIndicesStats().getFieldData().getMemorySizeInBytes(); - assertEquals("The percolator works with in-memory index and therefor shouldn't use field-data cache", 0L, fieldDataSize); - } - public void testPercolatorQueryViaMultiSearch() throws Exception { - createIndex("test", client().admin().indices().prepareCreate("test") + assertAcked(client().admin().indices().prepareCreate("test") .addMapping("type", "field1", "type=text", "query", "type=percolator") ); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java new file mode 100644 index 00000000000..020280670c4 --- /dev/null +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -0,0 +1,205 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.percolator; + +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.query.Operator; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.lookup.LeafDocLookup; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; + +public class PercolatorQuerySearchTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return Arrays.asList(PercolatorPlugin.class, CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + @Override + protected Map, Object>> pluginScripts() { + Map, Object>> scripts = new HashMap<>(); + scripts.put("1==1", vars -> Boolean.TRUE); + scripts.put("use_fielddata_please", vars -> { + LeafDocLookup leafDocLookup = (LeafDocLookup) vars.get("_doc"); + ScriptDocValues scriptDocValues = leafDocLookup.get("employees.name"); + return "virginia_potts".equals(scriptDocValues.get(0)); + }); + return scripts; + } + } + + public void testPercolateScriptQuery() throws IOException { + client().admin().indices().prepareCreate("index").addMapping("type", "query", "type=percolator").get(); + client().prepareIndex("index", "type", "1") + .setSource(jsonBuilder().startObject().field("query", QueryBuilders.scriptQuery( + new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1==1", Collections.emptyMap()))).endObject()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute().actionGet(); + SearchResponse response = client().prepareSearch("index") + .setQuery(new PercolateQueryBuilder("query", jsonBuilder().startObject().field("field1", "b").endObject().bytes(), + XContentType.JSON)) + .get(); + assertHitCount(response, 1); + assertSearchHits(response, "1"); + } + + public void testPercolateQueryWithNestedDocuments_doNotLeakBitsetCacheEntries() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder(); + mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject() + .startObject("query").field("type", "percolator").endObject() + .startObject("employee").field("type", "nested").startObject("properties") + .startObject("name").field("type", "text").endObject().endObject().endObject().endObject() + .endObject(); + createIndex("test", client().admin().indices().prepareCreate("test") + // to avoid normal document from being cached by BitsetFilterCache + .setSettings(Settings.builder().put(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING.getKey(), false)) + .addMapping("employee", mapping) + ); + client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject() + .field("query", QueryBuilders.nestedQuery("employee", + QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg) + ).endObject()) + .get(); + client().admin().indices().prepareRefresh().get(); + + for (int i = 0; i < 32; i++) { + SearchResponse response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", + XContentFactory.jsonBuilder() + .startObject().field("companyname", "stark") + .startArray("employee") + .startObject().field("name", "virginia potts").endObject() + .startObject().field("name", "tony stark").endObject() + .endArray() + .endObject().bytes(), XContentType.JSON)) + .addSort("_doc", SortOrder.ASC) + // size 0, because other wise load bitsets for normal document in FetchPhase#findRootDocumentIfNested(...) + .setSize(0) + .get(); + assertHitCount(response, 1); + } + + // We can't check via api... because BitsetCacheListener requires that it can extract shardId from index reader + // and for percolator it can't do that, but that means we don't keep track of + // memory for BitsetCache in case of percolator + long bitsetSize = client().admin().cluster().prepareClusterStats().get() + .getIndicesStats().getSegments().getBitsetMemoryInBytes(); + assertEquals("The percolator works with in-memory index and therefor shouldn't use bitset cache", 0L, bitsetSize); + } + + public void testPercolateQueryWithNestedDocuments_doLeakFieldDataCacheEntries() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder(); + mapping.startObject(); + { + mapping.startObject("properties"); + { + mapping.startObject("query"); + mapping.field("type", "percolator"); + mapping.endObject(); + } + { + mapping.startObject("companyname"); + mapping.field("type", "text"); + mapping.endObject(); + } + { + mapping.startObject("employees"); + mapping.field("type", "nested"); + { + mapping.startObject("properties"); + { + mapping.startObject("name"); + mapping.field("type", "text"); + mapping.field("fielddata", true); + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + createIndex("test", client().admin().indices().prepareCreate("test") + .addMapping("employee", mapping) + ); + Script script = new Script(ScriptType.INLINE, MockScriptPlugin.NAME, "use_fielddata_please", Collections.emptyMap()); + client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject() + .field("query", QueryBuilders.nestedQuery("employees", + QueryBuilders.scriptQuery(script), ScoreMode.Avg) + ).endObject()).get(); + client().admin().indices().prepareRefresh().get(); + XContentBuilder doc = jsonBuilder(); + doc.startObject(); + { + doc.field("companyname", "stark"); + doc.startArray("employees"); + { + doc.startObject(); + doc.field("name", "virginia_potts"); + doc.endObject(); + } + { + doc.startObject(); + doc.field("name", "tony_stark"); + doc.endObject(); + } + doc.endArray(); + } + doc.endObject(); + for (int i = 0; i < 32; i++) { + SearchResponse response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", doc.bytes(), XContentType.JSON)) + .addSort("_doc", SortOrder.ASC) + .get(); + assertHitCount(response, 1); + } + + long fieldDataSize = client().admin().cluster().prepareClusterStats().get() + .getIndicesStats().getFieldData().getMemorySizeInBytes(); + assertEquals("The percolator works with in-memory index and therefor shouldn't use field-data cache", 0L, fieldDataSize); + } + +}