From 00aad4c1237f9e3796688aa0ead28971ee0d9f31 Mon Sep 17 00:00:00 2001 From: kimchy Date: Tue, 25 Jan 2011 14:40:04 +0200 Subject: [PATCH] rename tests --- .../elasticsearch/search/SearchService.java | 4 + .../controller/SearchPhaseController.java | 3 + ...va => SingleShardEmbeddedSearchTests.java} | 83 +++++++++++++++---- ...va => ThreeShardsEmbeddedSearchTests.java} | 2 +- ...sUnbalancedShardsEmbeddedSearchTests.java} | 4 +- 5 files changed, 79 insertions(+), 17 deletions(-) rename modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/{SingleInstanceEmbeddedSearchTests.java => SingleShardEmbeddedSearchTests.java} (72%) rename modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/{TwoInstanceEmbeddedSearchTests.java => ThreeShardsEmbeddedSearchTests.java} (99%) rename modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/{TwoInstanceUnbalancedShardsEmbeddedSearchTests.java => ThreeShardsUnbalancedShardsEmbeddedSearchTests.java} (99%) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchService.java index 4f05069f6ab..6502496176c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchService.java @@ -423,6 +423,10 @@ public class SearchService extends AbstractLifecycleComponent { private static final int[] EMPTY_DOC_IDS = new int[0]; + /** + * Shortcut ids to load, we load only "from" and up to "size". The phase controller + * handles this as well since the result is always size * shards for Q_A_F + */ private void shortcutDocIdsToLoad(SearchContext context) { TopDocs topDocs = context.queryResult().topDocs(); if (topDocs.scoreDocs.length < context.from()) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java index ed01f0965a1..a3010dffa82 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java @@ -98,6 +98,7 @@ public class SearchPhaseController { int queueSize = queryResultProvider.queryResult().from() + queryResultProvider.queryResult().size(); if (queryResultProvider.includeFetch()) { // if we did both query and fetch on the same go, we have fetched all the docs from each shards already, use them... + // this is also important since we shortcut and fetch only docs from "from" and up to "size" queueSize *= results.size(); } PriorityQueue queue; @@ -173,6 +174,8 @@ public class SearchPhaseController { return EMPTY; } + // we only pop the first, this handles "from" nicely since the "from" are down the queue + // that we already fetched, so we are actually popping the "from" and up to "size" ShardDoc[] shardDocs = new ShardDoc[resultDocsSize]; for (int i = resultDocsSize - 1; i >= 0; i--) // put docs in array shardDocs[i] = (ShardDoc) queue.pop(); diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleInstanceEmbeddedSearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleShardEmbeddedSearchTests.java similarity index 72% rename from modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleInstanceEmbeddedSearchTests.java rename to modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleShardEmbeddedSearchTests.java index 151336ef88b..cf371f830c0 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleInstanceEmbeddedSearchTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/SingleShardEmbeddedSearchTests.java @@ -20,13 +20,11 @@ package org.elasticsearch.test.integration.search.embedded; import org.elasticsearch.client.Client; +import org.elasticsearch.common.collect.Sets; import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.internal.InternalNode; -import org.elasticsearch.search.Scroll; -import org.elasticsearch.search.SearchContextMissingException; -import org.elasticsearch.search.SearchService; -import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.*; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.controller.SearchPhaseController; import org.elasticsearch.search.controller.ShardDoc; @@ -40,12 +38,14 @@ import org.elasticsearch.search.fetch.QueryFetchSearchResult; import org.elasticsearch.search.internal.InternalSearchRequest; import org.elasticsearch.search.query.QuerySearchRequest; import org.elasticsearch.search.query.QuerySearchResult; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.util.Map; +import java.util.Set; import static org.elasticsearch.client.Requests.*; import static org.elasticsearch.common.collect.Lists.*; @@ -58,25 +58,28 @@ import static org.hamcrest.Matchers.*; /** * @author kimchy (shay.banon) */ -public class SingleInstanceEmbeddedSearchTests extends AbstractNodesTests { +public class SingleShardEmbeddedSearchTests extends AbstractNodesTests { private SearchService searchService; private SearchPhaseController searchPhaseController; + private Client client; + @BeforeClass public void createNodeAndInitWithData() throws Exception { startNode("server1"); + client = client("server1"); - client("server1").admin().indices().prepareCreate("test") + client.admin().indices().prepareCreate("test") .setSettings(settingsBuilder().put("number_of_shards", 1).put("number_of_replicas", 0)) .execute().actionGet(); client("server1").admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); - index(client("server1"), "1", "test1", 1); - index(client("server1"), "2", "test2", 2); - index(client("server1"), "3", "test3", 2); - index(client("server1"), "4", "test4", 2); - index(client("server1"), "5", "test5", 2); - client("server1").admin().indices().refresh(refreshRequest("test")).actionGet(); + index("1", "test1", 1); + index("2", "test2", 2); + index("3", "test3", 3); + index("4", "test4", 4); + index("5", "test5", 5); + client.admin().indices().refresh(refreshRequest("test")).actionGet(); searchService = ((InternalNode) node("server1")).injector().getInstance(SearchService.class); searchPhaseController = ((InternalNode) node("server1")).injector().getInstance(SearchPhaseController.class); @@ -125,6 +128,58 @@ public class SingleInstanceEmbeddedSearchTests extends AbstractNodesTests { assertThat(fetchResult.hits().hits()[0].type(), equalTo("type1")); } + @Test public void testQueryThenFetchIterateWithFrom() throws Exception { + QuerySearchResult queryResult = searchService.executeQueryPhase(searchRequest(searchSource().query(matchAllQuery()).from(0).size(2).sort("age", SortOrder.DESC))); + assertThat(queryResult.topDocs().totalHits, equalTo(5)); + + Set idsLoaded = Sets.newHashSet(); + + ShardDoc[] sortedShardList = searchPhaseController.sortDocs(newArrayList(queryResult)); + Map docIdsToLoad = searchPhaseController.docIdsToLoad(sortedShardList); + assertThat(docIdsToLoad.size(), equalTo(1)); + assertThat(docIdsToLoad.values().iterator().next().size(), equalTo(2)); + + FetchSearchResult fetchResult = searchService.executeFetchPhase(new FetchSearchRequest(queryResult.id(), docIdsToLoad.values().iterator().next())); + assertThat(fetchResult.hits().hits().length, equalTo(2)); + for (SearchHit hit : fetchResult.hits()) { + idsLoaded.add(hit.id()); + } + + // iterate to the next 2 + queryResult = searchService.executeQueryPhase(searchRequest(searchSource().query(matchAllQuery()).from(2).size(2).sort("age", SortOrder.DESC))); + assertThat(queryResult.topDocs().totalHits, equalTo(5)); + + sortedShardList = searchPhaseController.sortDocs(newArrayList(queryResult)); + docIdsToLoad = searchPhaseController.docIdsToLoad(sortedShardList); + assertThat(docIdsToLoad.size(), equalTo(1)); + assertThat(docIdsToLoad.values().iterator().next().size(), equalTo(2)); + + fetchResult = searchService.executeFetchPhase(new FetchSearchRequest(queryResult.id(), docIdsToLoad.values().iterator().next())); + assertThat(fetchResult.hits().hits().length, equalTo(2)); + for (SearchHit hit : fetchResult.hits()) { + idsLoaded.add(hit.id()); + } + + // iterate to the next 2 + queryResult = searchService.executeQueryPhase(searchRequest(searchSource().query(matchAllQuery()).from(4).size(2).sort("age", SortOrder.DESC))); + assertThat(queryResult.topDocs().totalHits, equalTo(5)); + + sortedShardList = searchPhaseController.sortDocs(newArrayList(queryResult)); + docIdsToLoad = searchPhaseController.docIdsToLoad(sortedShardList); + assertThat(docIdsToLoad.size(), equalTo(1)); + assertThat(docIdsToLoad.values().iterator().next().size(), equalTo(1)); + + fetchResult = searchService.executeFetchPhase(new FetchSearchRequest(queryResult.id(), docIdsToLoad.values().iterator().next())); + assertThat(fetchResult.hits().hits().length, equalTo(1)); + for (SearchHit hit : fetchResult.hits()) { + idsLoaded.add(hit.id()); + } + + // verify all ids were loaded + Set expectedIds = Sets.newHashSet("1", "2", "3", "4", "5"); + assertThat(idsLoaded, equalTo(expectedIds)); + } + @Test public void testQueryAndFetch() throws Exception { QueryFetchSearchResult result = searchService.executeFetchPhase(searchRequest(searchSource().query(termQuery("name", "test1")))); FetchSearchResult fetchResult = result.fetchResult(); @@ -157,7 +212,7 @@ public class SingleInstanceEmbeddedSearchTests extends AbstractNodesTests { .facet(FacetBuilders.queryFacet("age1", termQuery("age", 1))) .facet(FacetBuilders.queryFacet("age2", termQuery("age", 2))) )); - assertThat(queryResult.facets().facet(QueryFacet.class, "age2").count(), equalTo(4l)); + assertThat(queryResult.facets().facet(QueryFacet.class, "age2").count(), equalTo(1l)); assertThat(queryResult.facets().facet(QueryFacet.class, "age1").count(), equalTo(1l)); } @@ -186,7 +241,7 @@ public class SingleInstanceEmbeddedSearchTests extends AbstractNodesTests { return new InternalSearchRequest("test", 0, 1).source(builder.buildAsBytes()); } - private void index(Client client, String id, String nameValue, int age) { + private void index(String id, String nameValue, int age) { client.index(indexRequest("test").type("type1").id(id).source(source(id, nameValue, age))).actionGet(); } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/TwoInstanceEmbeddedSearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsEmbeddedSearchTests.java similarity index 99% rename from modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/TwoInstanceEmbeddedSearchTests.java rename to modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsEmbeddedSearchTests.java index 38705a17a3a..3e024df31c0 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/TwoInstanceEmbeddedSearchTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsEmbeddedSearchTests.java @@ -70,7 +70,7 @@ import static org.hamcrest.Matchers.*; /** * @author kimchy (shay.banon) */ -public class TwoInstanceEmbeddedSearchTests extends AbstractNodesTests { +public class ThreeShardsEmbeddedSearchTests extends AbstractNodesTests { private ClusterService clusterService; diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/TwoInstanceUnbalancedShardsEmbeddedSearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java similarity index 99% rename from modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/TwoInstanceUnbalancedShardsEmbeddedSearchTests.java rename to modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java index df250357b55..0a3b7952173 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/TwoInstanceUnbalancedShardsEmbeddedSearchTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java @@ -74,7 +74,7 @@ import static org.hamcrest.Matchers.*; /** * @author kimchy (shay.banon) */ -public class TwoInstanceUnbalancedShardsEmbeddedSearchTests extends AbstractNodesTests { +public class ThreeShardsUnbalancedShardsEmbeddedSearchTests extends AbstractNodesTests { private ClusterService clusterService; @@ -84,7 +84,7 @@ public class TwoInstanceUnbalancedShardsEmbeddedSearchTests extends AbstractNode @BeforeClass public void createNodeAndInitWithData() throws Exception { ImmutableSettings.Builder nodeSettings = ImmutableSettings.settingsBuilder() - .put("cluster.routing.operation.type", "org.elasticsearch.test.integration.search.embedded.TwoInstanceUnbalancedShardsEmbeddedSearchTests$UnevenOperationRoutingModule"); + .put("cluster.routing.operation.type", "org.elasticsearch.test.integration.search.embedded.ThreeShardsUnbalancedShardsEmbeddedSearchTests$UnevenOperationRoutingModule"); startNode("server1", nodeSettings); startNode("server2", nodeSettings);