honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adds code to help with IndicesRequestCacheIT failures (#33313) 

* Adds code to help with IndicesRequestCacheIT failures

Relates to #32827

* Adds comment

* Fixes test failure
This commit is contained in:
Colin Goodheart-Smithe 2018-09-03 14:54:17 +01:00 committed by GitHub
parent ccb2ad25cc
commit 0bf36253a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 112 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

304
server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
View File

@ -20,21 +20,25 @@
package org.elasticsearch.indices; package org.elasticsearch.indices;
import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.request.RequestCacheStats;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
import org.joda.time.DateTimeZone; import org.joda.time.DateTimeZone;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.List; import java.util.List;
import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram;
@ -107,41 +111,35 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
client.prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"), client.prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"),
client.prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27")); client.prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27"));
ensureSearchable("index"); ensureSearchable("index");
assertCacheState(client, "index", 0, 0);
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
equalTo(0L)); ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse);
equalTo(0L)); refresh();
ensureSearchable("index");
assertCacheState(client, "index", 0, 0);
final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).setPreFilterShardSize(Integer.MAX_VALUE) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).setPreFilterShardSize(Integer.MAX_VALUE).get();
.get(); ElasticsearchAssertions.assertAllSuccessful(r1);
assertSearchResponse(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(7L)); assertThat(r1.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 5);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(5L));
final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
.setPreFilterShardSize(Integer.MAX_VALUE).get(); .setPreFilterShardSize(Integer.MAX_VALUE).get();
assertSearchResponse(r2); ElasticsearchAssertions.assertAllSuccessful(r2);
assertThat(r2.getHits().getTotalHits(), equalTo(7L)); assertThat(r2.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 3, 7);
equalTo(3L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(7L));
final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).setPreFilterShardSize(Integer.MAX_VALUE) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).setPreFilterShardSize(Integer.MAX_VALUE)
.get(); .get();
assertSearchResponse(r3); ElasticsearchAssertions.assertAllSuccessful(r3);
assertThat(r3.getHits().getTotalHits(), equalTo(7L)); assertThat(r3.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 6, 9);
equalTo(6L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(9L));
} }
public void testQueryRewriteMissingValues() throws Exception { public void testQueryRewriteMissingValues() throws Exception {
@ -159,38 +157,33 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
client.prepareIndex("index", "type", "8").setSource("s", "2016-03-26"), client.prepareIndex("index", "type", "8").setSource("s", "2016-03-26"),
client.prepareIndex("index", "type", "9").setSource("s", "2016-03-27")); client.prepareIndex("index", "type", "9").setSource("s", "2016-03-27"));
ensureSearchable("index"); ensureSearchable("index");
assertCacheState(client, "index", 0, 0);
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
equalTo(0L)); ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse);
equalTo(0L)); refresh();
ensureSearchable("index");
assertCacheState(client, "index", 0, 0);
final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get(); .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get();
assertSearchResponse(r1); ElasticsearchAssertions.assertAllSuccessful(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(8L)); assertThat(r1.getHits().getTotalHits(), equalTo(8L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 1);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get(); .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get();
assertSearchResponse(r2); ElasticsearchAssertions.assertAllSuccessful(r2);
assertThat(r2.getHits().getTotalHits(), equalTo(8L)); assertThat(r2.getHits().getTotalHits(), equalTo(8L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 1, 1);
equalTo(1L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get(); .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get();
assertSearchResponse(r3); ElasticsearchAssertions.assertAllSuccessful(r3);
assertThat(r3.getHits().getTotalHits(), equalTo(8L)); assertThat(r3.getHits().getTotalHits(), equalTo(8L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 2, 1);
equalTo(2L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
} }
public void testQueryRewriteDates() throws Exception { public void testQueryRewriteDates() throws Exception {
@ -208,41 +201,36 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
client.prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00"), client.prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00"),
client.prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00")); client.prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00"));
ensureSearchable("index"); ensureSearchable("index");
assertCacheState(client, "index", 0, 0);
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
equalTo(0L)); ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse);
equalTo(0L)); refresh();
ensureSearchable("index");
assertCacheState(client, "index", 0, 0);
final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now")) .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
.get(); .get();
assertSearchResponse(r1); ElasticsearchAssertions.assertAllSuccessful(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(9L)); assertThat(r1.getHits().getTotalHits(), equalTo(9L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 1);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now")) .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
.get(); .get();
assertSearchResponse(r2); ElasticsearchAssertions.assertAllSuccessful(r2);
assertThat(r2.getHits().getTotalHits(), equalTo(9L)); assertThat(r2.getHits().getTotalHits(), equalTo(9L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 1, 1);
equalTo(1L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now")) .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
.get(); .get();
assertSearchResponse(r3); ElasticsearchAssertions.assertAllSuccessful(r3);
assertThat(r3.getHits().getTotalHits(), equalTo(9L)); assertThat(r3.getHits().getTotalHits(), equalTo(9L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 2, 1);
equalTo(2L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
} }
public void testQueryRewriteDatesWithNow() throws Exception { public void testQueryRewriteDatesWithNow() throws Exception {
@ -266,98 +254,47 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
client.prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7)), client.prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7)),
client.prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8))); client.prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8)));
ensureSearchable("index-1", "index-2", "index-3"); ensureSearchable("index-1", "index-2", "index-3");
assertCacheState(client, "index-1", 0, 0);
assertCacheState(client, "index-2", 0, 0);
assertCacheState(client, "index-3", 0, 0);
assertThat( // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index-1", "index-2", "index-3").setFlush(true)
equalTo(0L)); .get();
assertThat( ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse);
client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), refresh();
equalTo(0L)); ensureSearchable("index-1", "index-2", "index-3");
assertThat( assertCacheState(client, "index-1", 0, 0);
client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index-2", 0, 0);
equalTo(0L)); assertCacheState(client, "index-3", 0, 0);
assertThat(
client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
assertThat(
client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
equalTo(0L));
assertThat(
client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
final SearchResponse r1 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r1 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get(); .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
assertSearchResponse(r1); ElasticsearchAssertions.assertAllSuccessful(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(8L)); assertThat(r1.getHits().getTotalHits(), equalTo(8L));
assertThat( assertCacheState(client, "index-1", 0, 1);
client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index-2", 0, 1);
equalTo(0L));
assertThat(
client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
assertThat(
client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
equalTo(0L));
assertThat(
client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
// Because the query will INTERSECT with the 3rd index it will not be // Because the query will INTERSECT with the 3rd index it will not be
// rewritten and will still contain `now` so won't be recorded as a // rewritten and will still contain `now` so won't be recorded as a
// cache miss or cache hit since queries containing now can't be cached // cache miss or cache hit since queries containing now can't be cached
assertThat( assertCacheState(client, "index-3", 0, 0);
client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
equalTo(0L));
assertThat(
client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
final SearchResponse r2 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r2 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get(); .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
assertSearchResponse(r2); ElasticsearchAssertions.assertAllSuccessful(r2);
assertThat(r2.getHits().getTotalHits(), equalTo(8L)); assertThat(r2.getHits().getTotalHits(), equalTo(8L));
assertThat( assertCacheState(client, "index-1", 1, 1);
client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index-2", 1, 1);
equalTo(1L)); assertCacheState(client, "index-3", 0, 0);
assertThat(
client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
assertThat(
client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
equalTo(1L));
assertThat(
client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
assertThat(
client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
equalTo(0L));
assertThat(
client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
final SearchResponse r3 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r3 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get(); .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get();
assertSearchResponse(r3); ElasticsearchAssertions.assertAllSuccessful(r3);
assertThat(r3.getHits().getTotalHits(), equalTo(8L)); assertThat(r3.getHits().getTotalHits(), equalTo(8L));
assertThat( assertCacheState(client, "index-1", 2, 1);
client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index-2", 2, 1);
equalTo(2L)); assertCacheState(client, "index-3", 0, 0);
assertThat(
client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
assertThat(
client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
equalTo(2L));
assertThat(
client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
assertThat(
client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(),
equalTo(0L));
assertThat(
client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
} }
public void testCanCache() throws Exception { public void testCanCache() throws Exception {
@ -378,74 +315,60 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
client.prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"), client.prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"),
client.prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27")); client.prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27"));
ensureSearchable("index"); ensureSearchable("index");
assertCacheState(client, "index", 0, 0);
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
equalTo(0L)); ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse);
equalTo(0L)); refresh();
ensureSearchable("index");
assertCacheState(client, "index", 0, 0);
// If size > 0 we should no cache by default // If size > 0 we should no cache by default
final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1) final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1)
.setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).get(); .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).get();
assertSearchResponse(r1); ElasticsearchAssertions.assertAllSuccessful(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(7L)); assertThat(r1.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 0);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
// If search type is DFS_QUERY_THEN_FETCH we should not cache // If search type is DFS_QUERY_THEN_FETCH we should not cache
final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0) final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get(); .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get();
assertSearchResponse(r2); ElasticsearchAssertions.assertAllSuccessful(r2);
assertThat(r2.getHits().getTotalHits(), equalTo(7L)); assertThat(r2.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 0);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
// If search type is DFS_QUERY_THEN_FETCH we should not cache even if // If search type is DFS_QUERY_THEN_FETCH we should not cache even if
// the cache flag is explicitly set on the request // the cache flag is explicitly set on the request
final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0) final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0)
.setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get(); .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get();
assertSearchResponse(r3); ElasticsearchAssertions.assertAllSuccessful(r3);
assertThat(r3.getHits().getTotalHits(), equalTo(7L)); assertThat(r3.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 0);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
// If the request has an non-filter aggregation containing now we should not cache // If the request has an non-filter aggregation containing now we should not cache
final SearchResponse r5 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r5 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")) .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
.addAggregation(dateRange("foo").field("s").addRange("now-10y", "now")).get(); .addAggregation(dateRange("foo").field("s").addRange("now-10y", "now")).get();
assertSearchResponse(r5); ElasticsearchAssertions.assertAllSuccessful(r5);
assertThat(r5.getHits().getTotalHits(), equalTo(7L)); assertThat(r5.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 0);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
// If size > 1 and cache flag is set on the request we should cache // If size > 1 and cache flag is set on the request we should cache
final SearchResponse r6 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1) final SearchResponse r6 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1)
.setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).get(); .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).get();
assertSearchResponse(r6); ElasticsearchAssertions.assertAllSuccessful(r6);
assertThat(r6.getHits().getTotalHits(), equalTo(7L)); assertThat(r6.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 2);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(2L));
// If the request has a filter aggregation containing now we should cache since it gets rewritten // If the request has a filter aggregation containing now we should cache since it gets rewritten
final SearchResponse r4 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) final SearchResponse r4 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")) .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
.addAggregation(filter("foo", QueryBuilders.rangeQuery("s").from("now-10y").to("now"))).get(); .addAggregation(filter("foo", QueryBuilders.rangeQuery("s").from("now-10y").to("now"))).get();
assertSearchResponse(r4); ElasticsearchAssertions.assertAllSuccessful(r4);
assertThat(r4.getHits().getTotalHits(), equalTo(7L)); assertThat(r4.getHits().getTotalHits(), equalTo(7L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 4);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(4L));
} }
public void testCacheWithFilteredAlias() { public void testCacheWithFilteredAlias() {
@ -460,45 +383,42 @@ public class IndicesRequestCacheIT extends ESIntegTestCase {
client.prepareIndex("index", "type", "1").setRouting("1").setSource("created_at", client.prepareIndex("index", "type", "1").setRouting("1").setSource("created_at",
DateTimeFormatter.ISO_LOCAL_DATE.format(now)).get(); DateTimeFormatter.ISO_LOCAL_DATE.format(now)).get();
refresh(); refresh();
// Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
ElasticsearchAssertions.assertAllSuccessful(forceMergeResponse);
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 0);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(0L));
SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")).get(); .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")).get();
assertSearchResponse(r1); ElasticsearchAssertions.assertAllSuccessful(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(1L)); assertThat(r1.getHits().getTotalHits(), equalTo(1L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 0, 1);
equalTo(0L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0)
.setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")).get(); .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")).get();
assertSearchResponse(r1); ElasticsearchAssertions.assertAllSuccessful(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(1L)); assertThat(r1.getHits().getTotalHits(), equalTo(1L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 1, 1);
equalTo(1L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(1L));
r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get(); r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get();
assertSearchResponse(r1); ElasticsearchAssertions.assertAllSuccessful(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(1L)); assertThat(r1.getHits().getTotalHits(), equalTo(1L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 1, 2);
equalTo(1L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(2L));
r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get(); r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get();
assertSearchResponse(r1); ElasticsearchAssertions.assertAllSuccessful(r1);
assertThat(r1.getHits().getTotalHits(), equalTo(1L)); assertThat(r1.getHits().getTotalHits(), equalTo(1L));
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), assertCacheState(client, "index", 2, 2);
equalTo(2L)); }
assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(),
equalTo(2L)); private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
RequestCacheStats requestCacheStats = client.admin().indices().prepareStats(index).setRequestCache(true).get().getTotal()
.getRequestCache();
// Check the hit count and miss count together so if they are not
// correct we can see both values
assertEquals(Arrays.asList(expectedHits, expectedMisses),
Arrays.asList(requestCacheStats.getHitCount(), requestCacheStats.getMissCount()));
} }
} }