From a836496e57fa08ef80b0a858ffe52f58c892923b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 22 May 2014 15:26:00 +0200 Subject: [PATCH] [TESTS] Randomly disable the filter cache. Close #6280 --- .../indices/cache/CacheTests.java | 8 ++-- .../nested/SimpleNestedTests.java | 2 - .../child/SimpleChildQuerySearchTests.java | 2 +- .../search/query/SimpleQueryTests.java | 6 +-- .../scriptfilter/ScriptFilterSearchTests.java | 8 ++-- .../test/ExternalTestCluster.java | 5 +++ .../test/ImmutableTestCluster.java | 5 +++ .../org/elasticsearch/test/TestCluster.java | 14 ++++++- .../validate/SimpleValidateQueryTests.java | 41 +++++++++++-------- 9 files changed, 60 insertions(+), 31 deletions(-) diff --git a/src/test/java/org/elasticsearch/indices/cache/CacheTests.java b/src/test/java/org/elasticsearch/indices/cache/CacheTests.java index 41da58c7c62..3d5416dd8cb 100644 --- a/src/test/java/org/elasticsearch/indices/cache/CacheTests.java +++ b/src/test/java/org/elasticsearch/indices/cache/CacheTests.java @@ -61,9 +61,9 @@ public class CacheTests extends ElasticsearchIntegrationTest { SearchResponse searchResponse = client().prepareSearch().setQuery(filteredQuery(matchAllQuery(), FilterBuilders.termFilter("field", "value").cacheKey("test_key"))).execute().actionGet(); assertThat(searchResponse.getHits().getHits().length, equalTo(1)); nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); - assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), greaterThan(0l)); + assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L)); indicesStats = client().admin().indices().prepareStats("test").clear().setFilterCache(true).execute().actionGet(); - assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l)); + assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L)); client().admin().indices().prepareClearCache().setFilterKeys("test_key").execute().actionGet(); nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); @@ -152,13 +152,13 @@ public class CacheTests extends ElasticsearchIntegrationTest { nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true) .execute().actionGet(); assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0l)); - assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), greaterThan(0l)); + assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L)); indicesStats = client().admin().indices().prepareStats("test") .clear().setFieldData(true).setFilterCache(true) .execute().actionGet(); assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l)); - assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l)); + assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L)); client().admin().indices().prepareClearCache().execute().actionGet(); Thread.sleep(100); // Make sure the filter cache entries have been removed... diff --git a/src/test/java/org/elasticsearch/nested/SimpleNestedTests.java b/src/test/java/org/elasticsearch/nested/SimpleNestedTests.java index e2da85fd816..a51c7b2f6e9 100644 --- a/src/test/java/org/elasticsearch/nested/SimpleNestedTests.java +++ b/src/test/java/org/elasticsearch/nested/SimpleNestedTests.java @@ -21,8 +21,6 @@ package org.elasticsearch.nested; import org.apache.lucene.search.Explanation; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetResponse; diff --git a/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java b/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java index da0144f0d44..ab8dbe65e4d 100644 --- a/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java +++ b/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java @@ -1951,7 +1951,7 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest { // filter cache should not contain any thing, b/c has_child and has_parent can't be cached. statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get(); - assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(initialCacheSize)); + assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(initialCacheSize) : is(initialCacheSize)); } // https://github.com/elasticsearch/elasticsearch/issues/5783 diff --git a/src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java b/src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java index de3f0e7a9a2..6ca930e9c1c 100644 --- a/src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java +++ b/src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java @@ -2212,7 +2212,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest { // Now with rounding is used, so we must have something in filter cache statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get(); long filtercacheSize = statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(); - assertThat(filtercacheSize, greaterThan(0l)); + assertThat(filtercacheSize, cluster().hasFilterCache() ? greaterThan(0l) : is(0L)); searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.filteredQuery( @@ -2226,7 +2226,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest { // and because we use term filter, it is also added to filter cache, so it should contain more than before statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get(); - assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(filtercacheSize)); + assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(filtercacheSize) : is(filtercacheSize)); filtercacheSize = statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(); searchResponse = client().prepareSearch("test") @@ -2241,7 +2241,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest { // The range filter is now explicitly cached, so it now it is in the filter cache. statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get(); - assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(filtercacheSize)); + assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(filtercacheSize) : is(filtercacheSize)); } @Test diff --git a/src/test/java/org/elasticsearch/search/scriptfilter/ScriptFilterSearchTests.java b/src/test/java/org/elasticsearch/search/scriptfilter/ScriptFilterSearchTests.java index 29515cabdee..069745ba86f 100644 --- a/src/test/java/org/elasticsearch/search/scriptfilter/ScriptFilterSearchTests.java +++ b/src/test/java/org/elasticsearch/search/scriptfilter/ScriptFilterSearchTests.java @@ -124,7 +124,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest { .execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo(1l)); - assertThat(scriptCounter.get(), equalTo(3)); + assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 3 : 1)); scriptCounter.set(0); logger.info("running script filter the second time"); @@ -133,7 +133,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest { .execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo(1l)); - assertThat(scriptCounter.get(), equalTo(0)); + assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 0 : 1)); scriptCounter.set(0); logger.info("running script filter with new parameters"); @@ -142,7 +142,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest { .execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo(1l)); - assertThat(scriptCounter.get(), equalTo(3)); + assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 3 : 1)); scriptCounter.set(0); logger.info("running script filter with same parameters"); @@ -151,6 +151,6 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest { .execute().actionGet(); assertThat(response.getHits().totalHits(), equalTo(3l)); - assertThat(scriptCounter.get(), equalTo(0)); + assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 0 : 3)); } } \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/ExternalTestCluster.java b/src/test/java/org/elasticsearch/test/ExternalTestCluster.java index a46b3cd5ae2..6b94e346086 100644 --- a/src/test/java/org/elasticsearch/test/ExternalTestCluster.java +++ b/src/test/java/org/elasticsearch/test/ExternalTestCluster.java @@ -111,4 +111,9 @@ public final class ExternalTestCluster extends ImmutableTestCluster { public Iterator iterator() { return Lists.newArrayList(client).iterator(); } + + @Override + public boolean hasFilterCache() { + return true; // default + } } diff --git a/src/test/java/org/elasticsearch/test/ImmutableTestCluster.java b/src/test/java/org/elasticsearch/test/ImmutableTestCluster.java index b2ae726ac0b..aac70a77015 100644 --- a/src/test/java/org/elasticsearch/test/ImmutableTestCluster.java +++ b/src/test/java/org/elasticsearch/test/ImmutableTestCluster.java @@ -197,4 +197,9 @@ public abstract class ImmutableTestCluster implements Iterable { } } } + + /** + * Return whether or not this cluster can cache filters. + */ + public abstract boolean hasFilterCache(); } diff --git a/src/test/java/org/elasticsearch/test/TestCluster.java b/src/test/java/org/elasticsearch/test/TestCluster.java index 8c81b5b6b5c..2a10c947da4 100644 --- a/src/test/java/org/elasticsearch/test/TestCluster.java +++ b/src/test/java/org/elasticsearch/test/TestCluster.java @@ -58,6 +58,9 @@ import org.elasticsearch.common.util.BigArraysModule; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.index.cache.filter.FilterCacheModule; +import org.elasticsearch.index.cache.filter.none.NoneFilterCache; +import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache; import org.elasticsearch.index.engine.IndexEngineModule; import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder; import org.elasticsearch.node.Node; @@ -160,6 +163,8 @@ public final class TestCluster extends ImmutableTestCluster { private final ExecutorService executor; + private final boolean hasFilterCache; + public TestCluster(long clusterSeed, String clusterName) { this(clusterSeed, DEFAULT_MIN_NUM_DATA_NODES, DEFAULT_MAX_NUM_DATA_NODES, clusterName, NodeSettingsSource.EMPTY, DEFAULT_NUM_CLIENT_NODES, DEFAULT_ENABLE_RANDOM_BENCH_NODES); } @@ -228,6 +233,7 @@ public final class TestCluster extends ImmutableTestCluster { builder.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false); defaultSettings = builder.build(); executor = EsExecutors.newCached(1, TimeUnit.MINUTES, EsExecutors.daemonThreadFactory("test_" + clusterName)); + this.hasFilterCache = random.nextBoolean(); } public String getClusterName() { @@ -243,7 +249,8 @@ public final class TestCluster extends ImmutableTestCluster { private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) { Builder builder = ImmutableSettings.settingsBuilder().put(defaultSettings) - .put(getRandomNodeSettings(nodeSeed)); + .put(getRandomNodeSettings(nodeSeed)) + .put(FilterCacheModule.FilterCacheSettings.FILTER_CACHE_TYPE, hasFilterCache() ? WeightedFilterCache.class : NoneFilterCache.class); Settings settings = nodeSettingsSource.settings(nodeOrdinal); if (settings != null) { if (settings.get(ClusterName.SETTING) != null) { @@ -1278,6 +1285,11 @@ public final class TestCluster extends ImmutableTestCluster { return benchNodeAndClients().size(); } + @Override + public boolean hasFilterCache() { + return hasFilterCache; + } + private synchronized Collection dataNodeAndClients() { return Collections2.filter(nodes.values(), new DataNodePredicate()); } diff --git a/src/test/java/org/elasticsearch/validate/SimpleValidateQueryTests.java b/src/test/java/org/elasticsearch/validate/SimpleValidateQueryTests.java index 3d541bbe301..88378c9f9ce 100644 --- a/src/test/java/org/elasticsearch/validate/SimpleValidateQueryTests.java +++ b/src/test/java/org/elasticsearch/validate/SimpleValidateQueryTests.java @@ -77,6 +77,14 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest { assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("foo:1 AND")).execute().actionGet().isValid(), equalTo(false)); } + private static String filter(String uncachedFilter) { + String filter = uncachedFilter; + if (cluster().hasFilterCache()) { + filter = "cached(" + filter + ")"; + } + return filter; + } + @Test public void explainValidateQuery() throws Exception { createIndex("test"); @@ -110,12 +118,13 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest { assertThat(response.getQueryExplanation().get(0).getError(), containsString("Failed to parse")); assertThat(response.getQueryExplanation().get(0).getExplanation(), nullValue()); - assertExplanation(QueryBuilders.queryString("_id:1"), equalTo("filtered(ConstantScore(_uid:type1#1))->cache(_type:type1)")); + final String typeFilter = filter("_type:type1"); + assertExplanation(QueryBuilders.queryString("_id:1"), equalTo("filtered(ConstantScore(_uid:type1#1))->" + typeFilter)); assertExplanation(QueryBuilders.idsQuery("type1").addIds("1").addIds("2"), - equalTo("filtered(ConstantScore(_uid:type1#1 _uid:type1#2))->cache(_type:type1)")); + equalTo("filtered(ConstantScore(_uid:type1#1 _uid:type1#2))->" + typeFilter)); - assertExplanation(QueryBuilders.queryString("foo"), equalTo("filtered(_all:foo)->cache(_type:type1)")); + assertExplanation(QueryBuilders.queryString("foo"), equalTo("filtered(_all:foo)->" + typeFilter)); assertExplanation(QueryBuilders.filteredQuery( QueryBuilders.termQuery("foo", "1"), @@ -123,14 +132,14 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest { FilterBuilders.termFilter("bar", "2"), FilterBuilders.termFilter("baz", "3") ) - ), equalTo("filtered(filtered(foo:1)->cache(bar:[2 TO 2]) cache(baz:3))->cache(_type:type1)")); + ), equalTo("filtered(filtered(foo:1)->" + filter("bar:[2 TO 2]") + " " + filter("baz:3") + ")->" + typeFilter)); assertExplanation(QueryBuilders.filteredQuery( QueryBuilders.termQuery("foo", "1"), FilterBuilders.orFilter( FilterBuilders.termFilter("bar", "2") ) - ), equalTo("filtered(filtered(foo:1)->cache(bar:[2 TO 2]))->cache(_type:type1)")); + ), equalTo("filtered(filtered(foo:1)->" + filter("bar:[2 TO 2]") + ")->" + typeFilter)); assertExplanation(QueryBuilders.filteredQuery( QueryBuilders.matchAllQuery(), @@ -139,28 +148,28 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest { .addPoint(30, -80) .addPoint(20, -90) .addPoint(40, -70) // closing polygon - ), equalTo("filtered(ConstantScore(GeoPolygonFilter(pin.location, [[40.0, -70.0], [30.0, -80.0], [20.0, -90.0], [40.0, -70.0]])))->cache(_type:type1)")); + ), equalTo("filtered(ConstantScore(GeoPolygonFilter(pin.location, [[40.0, -70.0], [30.0, -80.0], [20.0, -90.0], [40.0, -70.0]])))->" + typeFilter)); assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoBoundingBoxFilter("pin.location") .topLeft(40, -80) .bottomRight(20, -70) - ), equalTo("filtered(ConstantScore(GeoBoundingBoxFilter(pin.location, [40.0, -80.0], [20.0, -70.0])))->cache(_type:type1)")); + ), equalTo("filtered(ConstantScore(GeoBoundingBoxFilter(pin.location, [40.0, -80.0], [20.0, -70.0])))->" + typeFilter)); assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceFilter("pin.location") .lat(10).lon(20).distance(15, DistanceUnit.DEFAULT).geoDistance(GeoDistance.PLANE) - ), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->cache(_type:type1)")); + ), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->" + typeFilter)); assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceFilter("pin.location") .lat(10).lon(20).distance(15, DistanceUnit.DEFAULT).geoDistance(GeoDistance.PLANE) - ), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->cache(_type:type1)")); + ), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->" + typeFilter)); assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceRangeFilter("pin.location") .lat(10).lon(20).from("15m").to("25m").geoDistance(GeoDistance.PLANE) - ), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [15.0 - 25.0], 10.0, 20.0)))->cache(_type:type1)")); + ), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [15.0 - 25.0], 10.0, 20.0)))->" + typeFilter)); assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceRangeFilter("pin.location") .lat(10).lon(20).from("15miles").to("25miles").geoDistance(GeoDistance.PLANE) - ), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [" + DistanceUnit.DEFAULT.convert(15.0, DistanceUnit.MILES) + " - " + DistanceUnit.DEFAULT.convert(25.0, DistanceUnit.MILES) + "], 10.0, 20.0)))->cache(_type:type1)")); + ), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [" + DistanceUnit.DEFAULT.convert(15.0, DistanceUnit.MILES) + " - " + DistanceUnit.DEFAULT.convert(25.0, DistanceUnit.MILES) + "], 10.0, 20.0)))->" + typeFilter)); assertExplanation(QueryBuilders.filteredQuery( QueryBuilders.termQuery("foo", "1"), @@ -168,13 +177,13 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest { FilterBuilders.termFilter("bar", "2"), FilterBuilders.termFilter("baz", "3") ) - ), equalTo("filtered(filtered(foo:1)->+cache(bar:[2 TO 2]) +cache(baz:3))->cache(_type:type1)")); + ), equalTo("filtered(filtered(foo:1)->+" + filter("bar:[2 TO 2]") + " +" + filter("baz:3") + ")->" + typeFilter)); assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.termsFilter("foo", "1", "2", "3")), - equalTo("filtered(ConstantScore(cache(foo:1 foo:2 foo:3)))->cache(_type:type1)")); + equalTo("filtered(ConstantScore(" + filter("foo:1 foo:2 foo:3") + "))->" + typeFilter)); assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.notFilter(FilterBuilders.termFilter("foo", "bar"))), - equalTo("filtered(ConstantScore(NotFilter(cache(foo:bar))))->cache(_type:type1)")); + equalTo("filtered(ConstantScore(NotFilter(" + filter("foo:bar") + ")))->" + typeFilter)); assertExplanation(QueryBuilders.filteredQuery( QueryBuilders.termQuery("foo", "1"), @@ -182,12 +191,12 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest { "child-type", QueryBuilders.matchQuery("foo", "1") ) - ), equalTo("filtered(filtered(foo:1)->CustomQueryWrappingFilter(child_filter[child-type/type1](filtered(foo:1)->cache(_type:child-type))))->cache(_type:type1)")); + ), equalTo("filtered(filtered(foo:1)->CustomQueryWrappingFilter(child_filter[child-type/type1](filtered(foo:1)->" + filter("_type:child-type") + ")))->" + typeFilter)); assertExplanation(QueryBuilders.filteredQuery( QueryBuilders.termQuery("foo", "1"), FilterBuilders.scriptFilter("true") - ), equalTo("filtered(filtered(foo:1)->ScriptFilter(true))->cache(_type:type1)")); + ), equalTo("filtered(filtered(foo:1)->ScriptFilter(true))->" + typeFilter)); }