[TESTS] Randomly disable the filter cache.

Close #6280

src/test/java/org/elasticsearch/indices/cache/CacheTests.java

@@ -61,9 +61,9 @@ public class CacheTests extends ElasticsearchIntegrationTest {
         SearchResponse searchResponse = client().prepareSearch().setQuery(filteredQuery(matchAllQuery(), FilterBuilders.termFilter("field", "value").cacheKey("test_key"))).execute().actionGet();
         assertThat(searchResponse.getHits().getHits().length, equalTo(1));
         nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
-        assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
+        assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
         indicesStats = client().admin().indices().prepareStats("test").clear().setFilterCache(true).execute().actionGet();
-        assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
+        assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
 
         client().admin().indices().prepareClearCache().setFilterKeys("test_key").execute().actionGet();
         nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
@@ -152,13 +152,13 @@ public class CacheTests extends ElasticsearchIntegrationTest {
         nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
                 .execute().actionGet();
         assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
-        assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
+        assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
 
         indicesStats = client().admin().indices().prepareStats("test")
                 .clear().setFieldData(true).setFilterCache(true)
                 .execute().actionGet();
         assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
-        assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
+        assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
 
         client().admin().indices().prepareClearCache().execute().actionGet();
         Thread.sleep(100); // Make sure the filter cache entries have been removed...

src/test/java/org/elasticsearch/nested/SimpleNestedTests.java

@@ -21,8 +21,6 @@ package org.elasticsearch.nested;
 
 import org.apache.lucene.search.Explanation;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetResponse;

src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java

@@ -1951,7 +1951,7 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest {
 
         // filter cache should not contain any thing, b/c has_child and has_parent can't be cached.
         statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get();
-        assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(initialCacheSize));
+        assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(initialCacheSize) : is(initialCacheSize));
     }
 
     // https://github.com/elasticsearch/elasticsearch/issues/5783

src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java

@@ -2212,7 +2212,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
         // Now with rounding is used, so we must have something in filter cache
         statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get();
         long filtercacheSize = statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes();
-        assertThat(filtercacheSize, greaterThan(0l));
+        assertThat(filtercacheSize, cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
 
         searchResponse = client().prepareSearch("test")
                 .setQuery(QueryBuilders.filteredQuery(
@@ -2226,7 +2226,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
 
         // and because we use term filter, it is also added to filter cache, so it should contain more than before
         statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get();
-        assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(filtercacheSize));
+        assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(filtercacheSize) : is(filtercacheSize));
         filtercacheSize = statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes();
 
         searchResponse = client().prepareSearch("test")
@@ -2241,7 +2241,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
 
         // The range filter is now explicitly cached, so it now it is in the filter cache.
         statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get();
-        assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(filtercacheSize));
+        assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(filtercacheSize) : is(filtercacheSize));
     }
 
     @Test

src/test/java/org/elasticsearch/search/scriptfilter/ScriptFilterSearchTests.java

@@ -124,7 +124,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
                 .execute().actionGet();
 
         assertThat(response.getHits().totalHits(), equalTo(1l));
-        assertThat(scriptCounter.get(), equalTo(3));
+        assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 3 : 1));
 
         scriptCounter.set(0);
         logger.info("running script filter the second time");
@@ -133,7 +133,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
                 .execute().actionGet();
 
         assertThat(response.getHits().totalHits(), equalTo(1l));
-        assertThat(scriptCounter.get(), equalTo(0));
+        assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 0 : 1));
 
         scriptCounter.set(0);
         logger.info("running script filter with new parameters");
@@ -142,7 +142,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
                 .execute().actionGet();
 
         assertThat(response.getHits().totalHits(), equalTo(1l));
-        assertThat(scriptCounter.get(), equalTo(3));
+        assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 3 : 1));
 
         scriptCounter.set(0);
         logger.info("running script filter with same parameters");
@@ -151,6 +151,6 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
                 .execute().actionGet();
 
         assertThat(response.getHits().totalHits(), equalTo(3l));
-        assertThat(scriptCounter.get(), equalTo(0));
+        assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 0 : 3));
     }
 }

src/test/java/org/elasticsearch/test/ExternalTestCluster.java

@@ -111,4 +111,9 @@ public final class ExternalTestCluster extends ImmutableTestCluster {
     public Iterator<Client> iterator() {
         return Lists.newArrayList(client).iterator();
     }
+
+    @Override
+    public boolean hasFilterCache() {
+        return true; // default
+    }
 }

src/test/java/org/elasticsearch/test/ImmutableTestCluster.java

@@ -197,4 +197,9 @@ public abstract class ImmutableTestCluster implements Iterable<Client> {
             }
         }
     }
+
+    /**
+     * Return whether or not this cluster can cache filters.
+     */
+    public abstract boolean hasFilterCache();
 }

src/test/java/org/elasticsearch/test/TestCluster.java

@@ -58,6 +58,9 @@ import org.elasticsearch.common.util.BigArraysModule;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.index.cache.filter.FilterCacheModule;
+import org.elasticsearch.index.cache.filter.none.NoneFilterCache;
+import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache;
 import org.elasticsearch.index.engine.IndexEngineModule;
 import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
 import org.elasticsearch.node.Node;
@@ -160,6 +163,8 @@ public final class TestCluster extends ImmutableTestCluster {
 
     private final ExecutorService executor;
 
+    private final boolean hasFilterCache;
+
     public TestCluster(long clusterSeed, String clusterName) {
         this(clusterSeed, DEFAULT_MIN_NUM_DATA_NODES, DEFAULT_MAX_NUM_DATA_NODES, clusterName, NodeSettingsSource.EMPTY, DEFAULT_NUM_CLIENT_NODES, DEFAULT_ENABLE_RANDOM_BENCH_NODES);
     }
@@ -228,6 +233,7 @@ public final class TestCluster extends ImmutableTestCluster {
         builder.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false);
         defaultSettings = builder.build();
         executor = EsExecutors.newCached(1, TimeUnit.MINUTES, EsExecutors.daemonThreadFactory("test_" + clusterName));
+        this.hasFilterCache = random.nextBoolean();
     }
 
     public String getClusterName() {
@@ -243,7 +249,8 @@ public final class TestCluster extends ImmutableTestCluster {
 
     private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) {
         Builder builder = ImmutableSettings.settingsBuilder().put(defaultSettings)
-                .put(getRandomNodeSettings(nodeSeed));
+                .put(getRandomNodeSettings(nodeSeed))
+                .put(FilterCacheModule.FilterCacheSettings.FILTER_CACHE_TYPE, hasFilterCache() ? WeightedFilterCache.class : NoneFilterCache.class);
         Settings settings = nodeSettingsSource.settings(nodeOrdinal);
         if (settings != null) {
             if (settings.get(ClusterName.SETTING) != null) {
@@ -1278,6 +1285,11 @@ public final class TestCluster extends ImmutableTestCluster {
         return benchNodeAndClients().size();
     }
 
+    @Override
+    public boolean hasFilterCache() {
+        return hasFilterCache;
+    }
+
     private synchronized Collection<NodeAndClient> dataNodeAndClients() {
         return Collections2.filter(nodes.values(), new DataNodePredicate());
     }

src/test/java/org/elasticsearch/validate/SimpleValidateQueryTests.java

@@ -77,6 +77,14 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
         assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("foo:1 AND")).execute().actionGet().isValid(), equalTo(false));
     }
 
+    private static String filter(String uncachedFilter) {
+        String filter = uncachedFilter;
+        if (cluster().hasFilterCache()) {
+            filter = "cached(" + filter + ")";
+        }
+        return filter;
+    }
+
     @Test
     public void explainValidateQuery() throws Exception {
         createIndex("test");
@@ -110,12 +118,13 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
         assertThat(response.getQueryExplanation().get(0).getError(), containsString("Failed to parse"));
         assertThat(response.getQueryExplanation().get(0).getExplanation(), nullValue());
 
-        assertExplanation(QueryBuilders.queryString("_id:1"), equalTo("filtered(ConstantScore(_uid:type1#1))->cache(_type:type1)"));
+        final String typeFilter = filter("_type:type1");
+        assertExplanation(QueryBuilders.queryString("_id:1"), equalTo("filtered(ConstantScore(_uid:type1#1))->" + typeFilter));
 
         assertExplanation(QueryBuilders.idsQuery("type1").addIds("1").addIds("2"),
-                equalTo("filtered(ConstantScore(_uid:type1#1 _uid:type1#2))->cache(_type:type1)"));
+                equalTo("filtered(ConstantScore(_uid:type1#1 _uid:type1#2))->" + typeFilter));
 
-        assertExplanation(QueryBuilders.queryString("foo"), equalTo("filtered(_all:foo)->cache(_type:type1)"));
+        assertExplanation(QueryBuilders.queryString("foo"), equalTo("filtered(_all:foo)->" + typeFilter));
 
         assertExplanation(QueryBuilders.filteredQuery(
                 QueryBuilders.termQuery("foo", "1"),
@@ -123,14 +132,14 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
                         FilterBuilders.termFilter("bar", "2"),
                         FilterBuilders.termFilter("baz", "3")
                 )
-        ), equalTo("filtered(filtered(foo:1)->cache(bar:[2 TO 2]) cache(baz:3))->cache(_type:type1)"));
+        ), equalTo("filtered(filtered(foo:1)->" + filter("bar:[2 TO 2]") + " " + filter("baz:3") + ")->" + typeFilter));
 
         assertExplanation(QueryBuilders.filteredQuery(
                 QueryBuilders.termQuery("foo", "1"),
                 FilterBuilders.orFilter(
                         FilterBuilders.termFilter("bar", "2")
                 )
-        ), equalTo("filtered(filtered(foo:1)->cache(bar:[2 TO 2]))->cache(_type:type1)"));
+        ), equalTo("filtered(filtered(foo:1)->" + filter("bar:[2 TO 2]") + ")->" + typeFilter));
 
         assertExplanation(QueryBuilders.filteredQuery(
                 QueryBuilders.matchAllQuery(),
@@ -139,28 +148,28 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
                         .addPoint(30, -80)
                         .addPoint(20, -90)
                         .addPoint(40, -70)    // closing polygon
-        ), equalTo("filtered(ConstantScore(GeoPolygonFilter(pin.location, [[40.0, -70.0], [30.0, -80.0], [20.0, -90.0], [40.0, -70.0]])))->cache(_type:type1)"));
+        ), equalTo("filtered(ConstantScore(GeoPolygonFilter(pin.location, [[40.0, -70.0], [30.0, -80.0], [20.0, -90.0], [40.0, -70.0]])))->" + typeFilter));
 
         assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoBoundingBoxFilter("pin.location")
                 .topLeft(40, -80)
                 .bottomRight(20, -70)
-        ), equalTo("filtered(ConstantScore(GeoBoundingBoxFilter(pin.location, [40.0, -80.0], [20.0, -70.0])))->cache(_type:type1)"));
+        ), equalTo("filtered(ConstantScore(GeoBoundingBoxFilter(pin.location, [40.0, -80.0], [20.0, -70.0])))->" + typeFilter));
 
         assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceFilter("pin.location")
                 .lat(10).lon(20).distance(15, DistanceUnit.DEFAULT).geoDistance(GeoDistance.PLANE)
-        ), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->cache(_type:type1)"));
+        ), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->" + typeFilter));
 
         assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceFilter("pin.location")
                 .lat(10).lon(20).distance(15, DistanceUnit.DEFAULT).geoDistance(GeoDistance.PLANE)
-        ), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->cache(_type:type1)"));
+        ), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->" + typeFilter));
 
         assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceRangeFilter("pin.location")
                 .lat(10).lon(20).from("15m").to("25m").geoDistance(GeoDistance.PLANE)
-        ), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [15.0 - 25.0], 10.0, 20.0)))->cache(_type:type1)"));
+        ), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [15.0 - 25.0], 10.0, 20.0)))->" + typeFilter));
 
         assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceRangeFilter("pin.location")
                 .lat(10).lon(20).from("15miles").to("25miles").geoDistance(GeoDistance.PLANE)
-        ), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [" + DistanceUnit.DEFAULT.convert(15.0, DistanceUnit.MILES) + " - " + DistanceUnit.DEFAULT.convert(25.0, DistanceUnit.MILES) + "], 10.0, 20.0)))->cache(_type:type1)"));
+        ), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [" + DistanceUnit.DEFAULT.convert(15.0, DistanceUnit.MILES) + " - " + DistanceUnit.DEFAULT.convert(25.0, DistanceUnit.MILES) + "], 10.0, 20.0)))->" + typeFilter));
 
         assertExplanation(QueryBuilders.filteredQuery(
                 QueryBuilders.termQuery("foo", "1"),
@@ -168,13 +177,13 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
                         FilterBuilders.termFilter("bar", "2"),
                         FilterBuilders.termFilter("baz", "3")
                 )
-        ), equalTo("filtered(filtered(foo:1)->+cache(bar:[2 TO 2]) +cache(baz:3))->cache(_type:type1)"));
+        ), equalTo("filtered(filtered(foo:1)->+" + filter("bar:[2 TO 2]") + " +" + filter("baz:3") + ")->" + typeFilter));
 
         assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.termsFilter("foo", "1", "2", "3")),
-                equalTo("filtered(ConstantScore(cache(foo:1 foo:2 foo:3)))->cache(_type:type1)"));
+                equalTo("filtered(ConstantScore(" + filter("foo:1 foo:2 foo:3") + "))->" + typeFilter));
 
         assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.notFilter(FilterBuilders.termFilter("foo", "bar"))),
-                equalTo("filtered(ConstantScore(NotFilter(cache(foo:bar))))->cache(_type:type1)"));
+                equalTo("filtered(ConstantScore(NotFilter(" + filter("foo:bar") + ")))->" + typeFilter));
 
         assertExplanation(QueryBuilders.filteredQuery(
                 QueryBuilders.termQuery("foo", "1"),
@@ -182,12 +191,12 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
                         "child-type",
                         QueryBuilders.matchQuery("foo", "1")
                 )
-        ), equalTo("filtered(filtered(foo:1)->CustomQueryWrappingFilter(child_filter[child-type/type1](filtered(foo:1)->cache(_type:child-type))))->cache(_type:type1)"));
+        ), equalTo("filtered(filtered(foo:1)->CustomQueryWrappingFilter(child_filter[child-type/type1](filtered(foo:1)->" + filter("_type:child-type") + ")))->" + typeFilter));
 
         assertExplanation(QueryBuilders.filteredQuery(
                 QueryBuilders.termQuery("foo", "1"),
                 FilterBuilders.scriptFilter("true")
-        ), equalTo("filtered(filtered(foo:1)->ScriptFilter(true))->cache(_type:type1)"));
+        ), equalTo("filtered(filtered(foo:1)->ScriptFilter(true))->" + typeFilter));
 
     }