honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test: Make many percolator integration tests real integration tests

This commit is contained in:
Martijn van Groningen 2017-06-27 17:44:10 +02:00
parent 03f952a838
commit c85ac402b0
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
2 changed files with 222 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
View File

@ -23,36 +23,19 @@ import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.MockScriptPlugin;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.lookup.LeafDocLookup;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ESSingleNodeTestCase;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Function;
import org.elasticsearch.test.ESIntegTestCase;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
@ -67,6 +50,7 @@ import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery;
import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
import static org.hamcrest.Matchers.containsString;
@ -75,44 +59,10 @@ import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.core.IsNull.notNullValue;
public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return Arrays.asList(PercolatorPlugin.class, CustomScriptPlugin.class);
}
public static class CustomScriptPlugin extends MockScriptPlugin {
@Override
protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
scripts.put("1==1", vars -> Boolean.TRUE);
scripts.put("use_fielddata_please", vars -> {
LeafDocLookup leafDocLookup = (LeafDocLookup) vars.get("_doc");
ScriptDocValues scriptDocValues = leafDocLookup.get("employees.name");
return "virginia_potts".equals(scriptDocValues.get(0));
});
return scripts;
}
}
public void testPercolateScriptQuery() throws IOException {
client().admin().indices().prepareCreate("index").addMapping("type", "query", "type=percolator").get();
client().prepareIndex("index", "type", "1")
.setSource(jsonBuilder().startObject().field("query", QueryBuilders.scriptQuery(
new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1==1", Collections.emptyMap()))).endObject())
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
.execute().actionGet();
SearchResponse response = client().prepareSearch("index")
.setQuery(new PercolateQueryBuilder("query", jsonBuilder().startObject().field("field1", "b").endObject().bytes(),
XContentType.JSON))
.get();
assertHitCount(response, 1);
assertSearchHits(response, "1");
}
public class PercolatorQuerySearchIT extends ESIntegTestCase {
public void testPercolatorQuery() throws Exception {
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator")
);
@ -160,7 +110,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
}
public void testPercolatorRangeQueries() throws Exception {
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=long", "field2", "type=double", "field3", "type=ip", "field4", "type=date",
"query", "type=percolator")
);
@ -269,7 +219,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
}
public void testPercolatorQueryExistingDocument() throws Exception {
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator")
);
@ -318,7 +268,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
}
public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception {
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "_source", "enabled=false", "field1", "type=keyword", "query", "type=percolator")
);
@ -340,7 +290,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
}
public void testPercolatorSpecificQueries() throws Exception {
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=text", "field2", "type=text", "query", "type=percolator")
);
@ -418,7 +368,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
} else if (randomBoolean()) {
fieldMapping.append(",index_options=offsets");
}
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", fieldMapping, "query", "type=percolator")
);
client().prepareIndex("test", "type", "1")
@ -461,7 +411,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
}
public void testTakePositionOffsetGapIntoAccount() throws Exception {
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field", "type=text,position_increment_gap=5", "query", "type=percolator")
);
client().prepareIndex("test", "type", "1")
@ -484,13 +434,13 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
public void testManyPercolatorFields() throws Exception {
String queryFieldName = randomAlphaOfLength(8);
createIndex("test1", client().admin().indices().prepareCreate("test1")
assertAcked(client().admin().indices().prepareCreate("test1")
.addMapping("type", queryFieldName, "type=percolator", "field", "type=keyword")
);
createIndex("test2", client().admin().indices().prepareCreate("test2")
assertAcked(client().admin().indices().prepareCreate("test2")
.addMapping("type", queryFieldName, "type=percolator", "second_query_field", "type=percolator", "field", "type=keyword")
);
createIndex("test3", client().admin().indices().prepareCreate("test3")
assertAcked(client().admin().indices().prepareCreate("test3")
.addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("field")
.field("type", "keyword")
@ -510,9 +460,9 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
public void testWithMultiplePercolatorFields() throws Exception {
String queryFieldName = randomAlphaOfLength(8);
createIndex("test1", client().admin().indices().prepareCreate("test1")
assertAcked(client().admin().indices().prepareCreate("test1")
.addMapping("type", queryFieldName, "type=percolator", "field", "type=keyword"));
createIndex("test2", client().admin().indices().prepareCreate("test2")
assertAcked(client().admin().indices().prepareCreate("test2")
.addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("field")
.field("type", "keyword")
@ -578,7 +528,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
.startObject("companyname").field("type", "text").endObject().startObject("employee").field("type", "nested")
.startObject("properties").startObject("name").field("type", "text").endObject().endObject().endObject().endObject()
.endObject();
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("employee", mapping)
);
client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject()
@ -628,125 +578,8 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
assertHitCount(response, 0);
}
public void testPercolateQueryWithNestedDocuments_doNotLeakBitsetCacheEntries() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject()
.startObject("query").field("type", "percolator").endObject()
.startObject("employee").field("type", "nested").startObject("properties")
.startObject("name").field("type", "text").endObject().endObject().endObject().endObject()
.endObject();
createIndex("test", client().admin().indices().prepareCreate("test")
// to avoid normal document from being cached by BitsetFilterCache
.setSettings(Settings.builder().put(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING.getKey(), false))
.addMapping("employee", mapping)
);
client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject()
.field("query", QueryBuilders.nestedQuery("employee",
QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg)
).endObject())
.get();
client().admin().indices().prepareRefresh().get();
for (int i = 0; i < 32; i++) {
SearchResponse response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query",
XContentFactory.jsonBuilder()
.startObject().field("companyname", "stark")
.startArray("employee")
.startObject().field("name", "virginia potts").endObject()
.startObject().field("name", "tony stark").endObject()
.endArray()
.endObject().bytes(), XContentType.JSON))
.addSort("_doc", SortOrder.ASC)
// size 0, because other wise load bitsets for normal document in FetchPhase#findRootDocumentIfNested(...)
.setSize(0)
.get();
assertHitCount(response, 1);
}
// We can't check via api... because BitsetCacheListener requires that it can extract shardId from index reader
// and for percolator it can't do that, but that means we don't keep track of
// memory for BitsetCache in case of percolator
long bitsetSize = client().admin().cluster().prepareClusterStats().get()
.getIndicesStats().getSegments().getBitsetMemoryInBytes();
assertEquals("The percolator works with in-memory index and therefor shouldn't use bitset cache", 0L, bitsetSize);
}
public void testPercolateQueryWithNestedDocuments_doLeakFieldDataCacheEntries() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject();
{
mapping.startObject("properties");
{
mapping.startObject("query");
mapping.field("type", "percolator");
mapping.endObject();
}
{
mapping.startObject("companyname");
mapping.field("type", "text");
mapping.endObject();
}
{
mapping.startObject("employees");
mapping.field("type", "nested");
{
mapping.startObject("properties");
{
mapping.startObject("name");
mapping.field("type", "text");
mapping.field("fielddata", true);
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
createIndex("test", client().admin().indices().prepareCreate("test")
.addMapping("employee", mapping)
);
Script script = new Script(ScriptType.INLINE, MockScriptPlugin.NAME, "use_fielddata_please", Collections.emptyMap());
client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject()
.field("query", QueryBuilders.nestedQuery("employees",
QueryBuilders.scriptQuery(script), ScoreMode.Avg)
).endObject()).get();
client().admin().indices().prepareRefresh().get();
XContentBuilder doc = jsonBuilder();
doc.startObject();
{
doc.field("companyname", "stark");
doc.startArray("employees");
{
doc.startObject();
doc.field("name", "virginia_potts");
doc.endObject();
}
{
doc.startObject();
doc.field("name", "tony_stark");
doc.endObject();
}
doc.endArray();
}
doc.endObject();
for (int i = 0; i < 32; i++) {
SearchResponse response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", doc.bytes(), XContentType.JSON))
.addSort("_doc", SortOrder.ASC)
.get();
assertHitCount(response, 1);
}
long fieldDataSize = client().admin().cluster().prepareClusterStats().get()
.getIndicesStats().getFieldData().getMemorySizeInBytes();
assertEquals("The percolator works with in-memory index and therefor shouldn't use field-data cache", 0L, fieldDataSize);
}
public void testPercolatorQueryViaMultiSearch() throws Exception {
createIndex("test", client().admin().indices().prepareCreate("test")
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type", "field1", "type=text", "query", "type=percolator")
);

205
modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java Normal file
View File

@ -0,0 +1,205 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.percolator;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.MockScriptPlugin;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.lookup.LeafDocLookup;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ESSingleNodeTestCase;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Function;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
public class PercolatorQuerySearchTests extends ESSingleNodeTestCase {
@Override
protected Collection<Class<? extends Plugin>> getPlugins() {
return Arrays.asList(PercolatorPlugin.class, CustomScriptPlugin.class);
}
public static class CustomScriptPlugin extends MockScriptPlugin {
@Override
protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
scripts.put("1==1", vars -> Boolean.TRUE);
scripts.put("use_fielddata_please", vars -> {
LeafDocLookup leafDocLookup = (LeafDocLookup) vars.get("_doc");
ScriptDocValues scriptDocValues = leafDocLookup.get("employees.name");
return "virginia_potts".equals(scriptDocValues.get(0));
});
return scripts;
}
}
public void testPercolateScriptQuery() throws IOException {
client().admin().indices().prepareCreate("index").addMapping("type", "query", "type=percolator").get();
client().prepareIndex("index", "type", "1")
.setSource(jsonBuilder().startObject().field("query", QueryBuilders.scriptQuery(
new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1==1", Collections.emptyMap()))).endObject())
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
.execute().actionGet();
SearchResponse response = client().prepareSearch("index")
.setQuery(new PercolateQueryBuilder("query", jsonBuilder().startObject().field("field1", "b").endObject().bytes(),
XContentType.JSON))
.get();
assertHitCount(response, 1);
assertSearchHits(response, "1");
}
public void testPercolateQueryWithNestedDocuments_doNotLeakBitsetCacheEntries() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject()
.startObject("query").field("type", "percolator").endObject()
.startObject("employee").field("type", "nested").startObject("properties")
.startObject("name").field("type", "text").endObject().endObject().endObject().endObject()
.endObject();
createIndex("test", client().admin().indices().prepareCreate("test")
// to avoid normal document from being cached by BitsetFilterCache
.setSettings(Settings.builder().put(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING.getKey(), false))
.addMapping("employee", mapping)
);
client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject()
.field("query", QueryBuilders.nestedQuery("employee",
QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg)
).endObject())
.get();
client().admin().indices().prepareRefresh().get();
for (int i = 0; i < 32; i++) {
SearchResponse response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query",
XContentFactory.jsonBuilder()
.startObject().field("companyname", "stark")
.startArray("employee")
.startObject().field("name", "virginia potts").endObject()
.startObject().field("name", "tony stark").endObject()
.endArray()
.endObject().bytes(), XContentType.JSON))
.addSort("_doc", SortOrder.ASC)
// size 0, because other wise load bitsets for normal document in FetchPhase#findRootDocumentIfNested(...)
.setSize(0)
.get();
assertHitCount(response, 1);
}
// We can't check via api... because BitsetCacheListener requires that it can extract shardId from index reader
// and for percolator it can't do that, but that means we don't keep track of
// memory for BitsetCache in case of percolator
long bitsetSize = client().admin().cluster().prepareClusterStats().get()
.getIndicesStats().getSegments().getBitsetMemoryInBytes();
assertEquals("The percolator works with in-memory index and therefor shouldn't use bitset cache", 0L, bitsetSize);
}
public void testPercolateQueryWithNestedDocuments_doLeakFieldDataCacheEntries() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder();
mapping.startObject();
{
mapping.startObject("properties");
{
mapping.startObject("query");
mapping.field("type", "percolator");
mapping.endObject();
}
{
mapping.startObject("companyname");
mapping.field("type", "text");
mapping.endObject();
}
{
mapping.startObject("employees");
mapping.field("type", "nested");
{
mapping.startObject("properties");
{
mapping.startObject("name");
mapping.field("type", "text");
mapping.field("fielddata", true);
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
createIndex("test", client().admin().indices().prepareCreate("test")
.addMapping("employee", mapping)
);
Script script = new Script(ScriptType.INLINE, MockScriptPlugin.NAME, "use_fielddata_please", Collections.emptyMap());
client().prepareIndex("test", "employee", "q1").setSource(jsonBuilder().startObject()
.field("query", QueryBuilders.nestedQuery("employees",
QueryBuilders.scriptQuery(script), ScoreMode.Avg)
).endObject()).get();
client().admin().indices().prepareRefresh().get();
XContentBuilder doc = jsonBuilder();
doc.startObject();
{
doc.field("companyname", "stark");
doc.startArray("employees");
{
doc.startObject();
doc.field("name", "virginia_potts");
doc.endObject();
}
{
doc.startObject();
doc.field("name", "tony_stark");
doc.endObject();
}
doc.endArray();
}
doc.endObject();
for (int i = 0; i < 32; i++) {
SearchResponse response = client().prepareSearch()
.setQuery(new PercolateQueryBuilder("query", doc.bytes(), XContentType.JSON))
.addSort("_doc", SortOrder.ASC)
.get();
assertHitCount(response, 1);
}
long fieldDataSize = client().admin().cluster().prepareClusterStats().get()
.getIndicesStats().getFieldData().getMemorySizeInBytes();
assertEquals("The percolator works with in-memory index and therefor shouldn't use field-data cache", 0L, fieldDataSize);
}
}