honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[TESTS] Randomly disable the filter cache. 

Close #6280
This commit is contained in:
Adrien Grand 2014-05-22 15:26:00 +02:00
parent 6e49256fa8
commit a836496e57
9 changed files with 60 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/test/java/org/elasticsearch/indices/cache/CacheTests.java vendored
View File

@ -61,9 +61,9 @@ public class CacheTests extends ElasticsearchIntegrationTest {
SearchResponse searchResponse = client().prepareSearch().setQuery(filteredQuery(matchAllQuery(), FilterBuilders.termFilter("field", "value").cacheKey("test_key"))).execute().actionGet();
assertThat(searchResponse.getHits().getHits().length, equalTo(1));
nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
indicesStats = client().admin().indices().prepareStats("test").clear().setFilterCache(true).execute().actionGet();
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
client().admin().indices().prepareClearCache().setFilterKeys("test_key").execute().actionGet();
nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
@ -152,13 +152,13 @@ public class CacheTests extends ElasticsearchIntegrationTest {
nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
.execute().actionGet();
assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
assertThat(nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
indicesStats = client().admin().indices().prepareStats("test")
.clear().setFieldData(true).setFilterCache(true)
.execute().actionGet();
assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));
assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
client().admin().indices().prepareClearCache().execute().actionGet();
Thread.sleep(100); // Make sure the filter cache entries have been removed...

2
src/test/java/org/elasticsearch/nested/SimpleNestedTests.java
View File

@ -21,8 +21,6 @@ package org.elasticsearch.nested;
import org.apache.lucene.search.Explanation;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetResponse;

2
src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java
View File

@ -1951,7 +1951,7 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest {
// filter cache should not contain any thing, b/c has_child and has_parent can't be cached.
statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get();
assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(initialCacheSize));
assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(initialCacheSize) : is(initialCacheSize));
}
// https://github.com/elasticsearch/elasticsearch/issues/5783

6
src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java
View File

@ -2212,7 +2212,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
// Now with rounding is used, so we must have something in filter cache
statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get();
long filtercacheSize = statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes();
assertThat(filtercacheSize, greaterThan(0l));
assertThat(filtercacheSize, cluster().hasFilterCache() ? greaterThan(0l) : is(0L));
searchResponse = client().prepareSearch("test")
.setQuery(QueryBuilders.filteredQuery(
@ -2226,7 +2226,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
// and because we use term filter, it is also added to filter cache, so it should contain more than before
statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get();
assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(filtercacheSize));
assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(filtercacheSize) : is(filtercacheSize));
filtercacheSize = statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes();
searchResponse = client().prepareSearch("test")
@ -2241,7 +2241,7 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
// The range filter is now explicitly cached, so it now it is in the filter cache.
statsResponse = client().admin().indices().prepareStats("test").clear().setFilterCache(true).get();
assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(filtercacheSize));
assertThat(statsResponse.getIndex("test").getTotal().getFilterCache().getMemorySizeInBytes(), cluster().hasFilterCache() ? greaterThan(filtercacheSize) : is(filtercacheSize));
}
@Test

8
src/test/java/org/elasticsearch/search/scriptfilter/ScriptFilterSearchTests.java
View File

@ -124,7 +124,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo(1l));
assertThat(scriptCounter.get(), equalTo(3));
assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 3 : 1));
scriptCounter.set(0);
logger.info("running script filter the second time");
@ -133,7 +133,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo(1l));
assertThat(scriptCounter.get(), equalTo(0));
assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 0 : 1));
scriptCounter.set(0);
logger.info("running script filter with new parameters");
@ -142,7 +142,7 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo(1l));
assertThat(scriptCounter.get(), equalTo(3));
assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 3 : 1));
scriptCounter.set(0);
logger.info("running script filter with same parameters");
@ -151,6 +151,6 @@ public class ScriptFilterSearchTests extends ElasticsearchIntegrationTest {
.execute().actionGet();
assertThat(response.getHits().totalHits(), equalTo(3l));
assertThat(scriptCounter.get(), equalTo(0));
assertThat(scriptCounter.get(), equalTo(cluster().hasFilterCache() ? 0 : 3));
}
}

5
src/test/java/org/elasticsearch/test/ExternalTestCluster.java
View File

@ -111,4 +111,9 @@ public final class ExternalTestCluster extends ImmutableTestCluster {
public Iterator<Client> iterator() {
return Lists.newArrayList(client).iterator();
}
@Override
public boolean hasFilterCache() {
return true; // default
}
}

5
src/test/java/org/elasticsearch/test/ImmutableTestCluster.java
View File

@ -197,4 +197,9 @@ public abstract class ImmutableTestCluster implements Iterable<Client> {
}
}
}
/**
* Return whether or not this cluster can cache filters.
*/
public abstract boolean hasFilterCache();
}

14
src/test/java/org/elasticsearch/test/TestCluster.java
View File

@ -58,6 +58,9 @@ import org.elasticsearch.common.util.BigArraysModule;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.http.HttpServerTransport;
import org.elasticsearch.index.cache.filter.FilterCacheModule;
import org.elasticsearch.index.cache.filter.none.NoneFilterCache;
import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache;
import org.elasticsearch.index.engine.IndexEngineModule;
import org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder;
import org.elasticsearch.node.Node;
@ -160,6 +163,8 @@ public final class TestCluster extends ImmutableTestCluster {
private final ExecutorService executor;
private final boolean hasFilterCache;
public TestCluster(long clusterSeed, String clusterName) {
this(clusterSeed, DEFAULT_MIN_NUM_DATA_NODES, DEFAULT_MAX_NUM_DATA_NODES, clusterName, NodeSettingsSource.EMPTY, DEFAULT_NUM_CLIENT_NODES, DEFAULT_ENABLE_RANDOM_BENCH_NODES);
}
@ -228,6 +233,7 @@ public final class TestCluster extends ImmutableTestCluster {
builder.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false);
defaultSettings = builder.build();
executor = EsExecutors.newCached(1, TimeUnit.MINUTES, EsExecutors.daemonThreadFactory("test_" + clusterName));
this.hasFilterCache = random.nextBoolean();
}
public String getClusterName() {
@ -243,7 +249,8 @@ public final class TestCluster extends ImmutableTestCluster {
private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) {
Builder builder = ImmutableSettings.settingsBuilder().put(defaultSettings)
.put(getRandomNodeSettings(nodeSeed));
.put(getRandomNodeSettings(nodeSeed))
.put(FilterCacheModule.FilterCacheSettings.FILTER_CACHE_TYPE, hasFilterCache() ? WeightedFilterCache.class : NoneFilterCache.class);
Settings settings = nodeSettingsSource.settings(nodeOrdinal);
if (settings != null) {
if (settings.get(ClusterName.SETTING) != null) {
@ -1278,6 +1285,11 @@ public final class TestCluster extends ImmutableTestCluster {
return benchNodeAndClients().size();
}
@Override
public boolean hasFilterCache() {
return hasFilterCache;
}
private synchronized Collection<NodeAndClient> dataNodeAndClients() {
return Collections2.filter(nodes.values(), new DataNodePredicate());
}

41
src/test/java/org/elasticsearch/validate/SimpleValidateQueryTests.java
View File

@ -77,6 +77,14 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("foo:1 AND")).execute().actionGet().isValid(), equalTo(false));
}
private static String filter(String uncachedFilter) {
String filter = uncachedFilter;
if (cluster().hasFilterCache()) {
filter = "cached(" + filter + ")";
}
return filter;
}
@Test
public void explainValidateQuery() throws Exception {
createIndex("test");
@ -110,12 +118,13 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
assertThat(response.getQueryExplanation().get(0).getError(), containsString("Failed to parse"));
assertThat(response.getQueryExplanation().get(0).getExplanation(), nullValue());
assertExplanation(QueryBuilders.queryString("_id:1"), equalTo("filtered(ConstantScore(_uid:type1#1))->cache(_type:type1)"));
final String typeFilter = filter("_type:type1");
assertExplanation(QueryBuilders.queryString("_id:1"), equalTo("filtered(ConstantScore(_uid:type1#1))->" + typeFilter));
assertExplanation(QueryBuilders.idsQuery("type1").addIds("1").addIds("2"),
equalTo("filtered(ConstantScore(_uid:type1#1 _uid:type1#2))->cache(_type:type1)"));
equalTo("filtered(ConstantScore(_uid:type1#1 _uid:type1#2))->" + typeFilter));
assertExplanation(QueryBuilders.queryString("foo"), equalTo("filtered(_all:foo)->cache(_type:type1)"));
assertExplanation(QueryBuilders.queryString("foo"), equalTo("filtered(_all:foo)->" + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
@ -123,14 +132,14 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
FilterBuilders.termFilter("bar", "2"),
FilterBuilders.termFilter("baz", "3")
)
), equalTo("filtered(filtered(foo:1)->cache(bar:[2 TO 2]) cache(baz:3))->cache(_type:type1)"));
), equalTo("filtered(filtered(foo:1)->" + filter("bar:[2 TO 2]") + " " + filter("baz:3") + ")->" + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
FilterBuilders.orFilter(
FilterBuilders.termFilter("bar", "2")
)
), equalTo("filtered(filtered(foo:1)->cache(bar:[2 TO 2]))->cache(_type:type1)"));
), equalTo("filtered(filtered(foo:1)->" + filter("bar:[2 TO 2]") + ")->" + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.matchAllQuery(),
@ -139,28 +148,28 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
.addPoint(30, -80)
.addPoint(20, -90)
.addPoint(40, -70) // closing polygon
), equalTo("filtered(ConstantScore(GeoPolygonFilter(pin.location, [[40.0, -70.0], [30.0, -80.0], [20.0, -90.0], [40.0, -70.0]])))->cache(_type:type1)"));
), equalTo("filtered(ConstantScore(GeoPolygonFilter(pin.location, [[40.0, -70.0], [30.0, -80.0], [20.0, -90.0], [40.0, -70.0]])))->" + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoBoundingBoxFilter("pin.location")
.topLeft(40, -80)
.bottomRight(20, -70)
), equalTo("filtered(ConstantScore(GeoBoundingBoxFilter(pin.location, [40.0, -80.0], [20.0, -70.0])))->cache(_type:type1)"));
), equalTo("filtered(ConstantScore(GeoBoundingBoxFilter(pin.location, [40.0, -80.0], [20.0, -70.0])))->" + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceFilter("pin.location")
.lat(10).lon(20).distance(15, DistanceUnit.DEFAULT).geoDistance(GeoDistance.PLANE)
), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->cache(_type:type1)"));
), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->" + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceFilter("pin.location")
.lat(10).lon(20).distance(15, DistanceUnit.DEFAULT).geoDistance(GeoDistance.PLANE)
), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->cache(_type:type1)"));
), equalTo("filtered(ConstantScore(GeoDistanceFilter(pin.location, PLANE, 15.0, 10.0, 20.0)))->" + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceRangeFilter("pin.location")
.lat(10).lon(20).from("15m").to("25m").geoDistance(GeoDistance.PLANE)
), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [15.0 - 25.0], 10.0, 20.0)))->cache(_type:type1)"));
), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [15.0 - 25.0], 10.0, 20.0)))->" + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.geoDistanceRangeFilter("pin.location")
.lat(10).lon(20).from("15miles").to("25miles").geoDistance(GeoDistance.PLANE)
), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [" + DistanceUnit.DEFAULT.convert(15.0, DistanceUnit.MILES) + " - " + DistanceUnit.DEFAULT.convert(25.0, DistanceUnit.MILES) + "], 10.0, 20.0)))->cache(_type:type1)"));
), equalTo("filtered(ConstantScore(GeoDistanceRangeFilter(pin.location, PLANE, [" + DistanceUnit.DEFAULT.convert(15.0, DistanceUnit.MILES) + " - " + DistanceUnit.DEFAULT.convert(25.0, DistanceUnit.MILES) + "], 10.0, 20.0)))->" + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
@ -168,13 +177,13 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
FilterBuilders.termFilter("bar", "2"),
FilterBuilders.termFilter("baz", "3")
)
), equalTo("filtered(filtered(foo:1)->+cache(bar:[2 TO 2]) +cache(baz:3))->cache(_type:type1)"));
), equalTo("filtered(filtered(foo:1)->+" + filter("bar:[2 TO 2]") + " +" + filter("baz:3") + ")->" + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.termsFilter("foo", "1", "2", "3")),
equalTo("filtered(ConstantScore(cache(foo:1 foo:2 foo:3)))->cache(_type:type1)"));
equalTo("filtered(ConstantScore(" + filter("foo:1 foo:2 foo:3") + "))->" + typeFilter));
assertExplanation(QueryBuilders.constantScoreQuery(FilterBuilders.notFilter(FilterBuilders.termFilter("foo", "bar"))),
equalTo("filtered(ConstantScore(NotFilter(cache(foo:bar))))->cache(_type:type1)"));
equalTo("filtered(ConstantScore(NotFilter(" + filter("foo:bar") + ")))->" + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
@ -182,12 +191,12 @@ public class SimpleValidateQueryTests extends ElasticsearchIntegrationTest {
"child-type",
QueryBuilders.matchQuery("foo", "1")
)
), equalTo("filtered(filtered(foo:1)->CustomQueryWrappingFilter(child_filter[child-type/type1](filtered(foo:1)->cache(_type:child-type))))->cache(_type:type1)"));
), equalTo("filtered(filtered(foo:1)->CustomQueryWrappingFilter(child_filter[child-type/type1](filtered(foo:1)->" + filter("_type:child-type") + ")))->" + typeFilter));
assertExplanation(QueryBuilders.filteredQuery(
QueryBuilders.termQuery("foo", "1"),
FilterBuilders.scriptFilter("true")
), equalTo("filtered(filtered(foo:1)->ScriptFilter(true))->cache(_type:type1)"));
), equalTo("filtered(filtered(foo:1)->ScriptFilter(true))->" + typeFilter));
}