From 06ec935a08753801f200d989cac0a5401b8ce41c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 7 Sep 2015 17:44:01 +0200 Subject: [PATCH] Fix x-plugins following count/scan removal from elasticsearch-core. Original commit: elastic/x-pack-elasticsearch@ee43991a0a496c4078c53800d88a8a44fde45c4e --- .../authz/InternalAuthorizationService.java | 1 - .../InternalAuthorizationServiceTests.java | 3 -- .../org/elasticsearch/transport/handlers | 4 +-- .../execution/TriggeredWatchStore.java | 28 ++++++++--------- .../watcher/support/WatcherUtils.java | 3 -- .../watcher/watch/WatchStore.java | 30 +++++++++---------- .../input/search/SearchInputTests.java | 20 ++----------- .../AbstractWatcherIntegrationTestCase.java | 4 +-- .../search/SearchTransformTests.java | 22 -------------- 9 files changed, 32 insertions(+), 83 deletions(-) diff --git a/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java b/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java index b4d61efcf33..12e7014f07b 100644 --- a/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java +++ b/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java @@ -223,7 +223,6 @@ public class InternalAuthorizationService extends AbstractComponent implements A private static boolean isScrollRelatedAction(String action) { return action.equals(SearchScrollAction.NAME) || - action.equals(SearchServiceTransportAction.SCAN_SCROLL_ACTION_NAME) || action.equals(SearchServiceTransportAction.FETCH_ID_SCROLL_ACTION_NAME) || action.equals(SearchServiceTransportAction.QUERY_FETCH_SCROLL_ACTION_NAME) || action.equals(SearchServiceTransportAction.QUERY_SCROLL_ACTION_NAME) || diff --git a/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java b/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java index 1d7d2115af4..9ac3b746d71 100644 --- a/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/authz/InternalAuthorizationServiceTests.java @@ -175,9 +175,6 @@ public class InternalAuthorizationServiceTests extends ESTestCase { internalAuthorizationService.authorize(user, SearchServiceTransportAction.CLEAR_SCROLL_CONTEXTS_ACTION_NAME, request); verify(auditTrail).accessGranted(user, SearchServiceTransportAction.CLEAR_SCROLL_CONTEXTS_ACTION_NAME, request); - internalAuthorizationService.authorize(user, SearchServiceTransportAction.SCAN_SCROLL_ACTION_NAME, request); - verify(auditTrail).accessGranted(user, SearchServiceTransportAction.SCAN_SCROLL_ACTION_NAME, request); - internalAuthorizationService.authorize(user, SearchServiceTransportAction.FETCH_ID_SCROLL_ACTION_NAME, request); verify(auditTrail).accessGranted(user, SearchServiceTransportAction.FETCH_ID_SCROLL_ACTION_NAME, request); diff --git a/shield/src/test/resources/org/elasticsearch/transport/handlers b/shield/src/test/resources/org/elasticsearch/transport/handlers index 32433ef9186..ebf5177a0f0 100644 --- a/shield/src/test/resources/org/elasticsearch/transport/handlers +++ b/shield/src/test/resources/org/elasticsearch/transport/handlers @@ -40,8 +40,6 @@ indices:data/read/search[phase/query/id] indices:data/read/search[phase/query/query+fetch] indices:data/read/search[phase/query/scroll] indices:data/read/search[phase/query] -indices:data/read/search[phase/scan/scroll] -indices:data/read/search[phase/scan] indices:data/read/suggest[s] indices:data/read/tv[s] indices:data/write/bulk[s] @@ -92,4 +90,4 @@ internal:indices/flush/synced/in_flight internal:indices/flush/synced/pre internal:indices/flush/synced/sync internal:admin/repository/verify -internal:plugin/license/cluster/register_trial_license \ No newline at end of file +internal:plugin/license/cluster/register_trial_license diff --git a/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java b/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java index 55118b8edbe..e44abdf2bec 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/execution/TriggeredWatchStore.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.watcher.support.init.proxy.ClientProxy; import java.io.IOException; @@ -250,21 +251,18 @@ public class TriggeredWatchStore extends AbstractComponent { throw illegalState("scan search was supposed to run on [{}] shards, but ran on [{}] shards", numPrimaryShards, response.getSuccessfulShards()); } - if (response.getHits().getTotalHits() > 0) { - response = client.searchScroll(response.getScrollId(), scrollTimeout); - while (response.getHits().hits().length != 0) { - for (SearchHit sh : response.getHits()) { - String id = sh.getId(); - try { - TriggeredWatch triggeredWatch = triggeredWatchParser.parse(id, sh.version(), sh.getSourceRef()); - logger.debug("loaded triggered watch [{}/{}/{}]", sh.index(), sh.type(), sh.id()); - triggeredWatches.add(triggeredWatch); - } catch (Exception e) { - logger.error("couldn't load triggered watch [{}], ignoring it...", e, id); - } + while (response.getHits().hits().length != 0) { + for (SearchHit sh : response.getHits()) { + String id = sh.getId(); + try { + TriggeredWatch triggeredWatch = triggeredWatchParser.parse(id, sh.version(), sh.getSourceRef()); + logger.debug("loaded triggered watch [{}/{}/{}]", sh.index(), sh.type(), sh.id()); + triggeredWatches.add(triggeredWatch); + } catch (Exception e) { + logger.error("couldn't load triggered watch [{}], ignoring it...", e, id); } - response = client.searchScroll(response.getScrollId(), scrollTimeout); } + response = client.searchScroll(response.getScrollId(), scrollTimeout); } } finally { client.clearScroll(response.getScrollId()); @@ -274,11 +272,11 @@ public class TriggeredWatchStore extends AbstractComponent { private SearchRequest createScanSearchRequest() { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() - .size(scrollSize); + .size(scrollSize) + .sort(SortBuilders.fieldSort("_doc")); SearchRequest searchRequest = new SearchRequest(INDEX_NAME); searchRequest.source(sourceBuilder); - searchRequest.searchType(SearchType.SCAN); searchRequest.types(DOC_TYPE); searchRequest.scroll(scrollTimeout); searchRequest.preference("_primary"); diff --git a/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java b/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java index 7c1fdd9e0c3..1926d47adf0 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherUtils.java @@ -231,9 +231,6 @@ public final class WatcherUtils { searchRequest.types(Strings.delimitedListToStringArray(typesStr, ",", " \t")); } else if (ParseFieldMatcher.STRICT.match(currentFieldName, SEARCH_TYPE_FIELD)) { searchType = SearchType.fromString(parser.text().toLowerCase(Locale.ROOT), ParseFieldMatcher.EMPTY); - if (searchType == SearchType.SCAN){ - throw new ElasticsearchParseException("could not read search request. value [" + searchType.name() + "] is not supported for field [" + SEARCH_TYPE_FIELD.getPreferredName() + "]" ); - } } else { throw new ElasticsearchParseException("could not read search request. unexpected string field [" + currentFieldName + "]"); } diff --git a/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java b/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java index 969814e786f..5261817ccca 100644 --- a/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java +++ b/watcher/src/main/java/org/elasticsearch/watcher/watch/WatchStore.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.watcher.support.init.proxy.ClientProxy; import java.io.IOException; @@ -230,10 +231,10 @@ public class WatchStore extends AbstractComponent { SearchRequest searchRequest = new SearchRequest(INDEX) .types(DOC_TYPE) .preference("_primary") - .searchType(SearchType.SCAN) .scroll(scrollTimeout) .source(new SearchSourceBuilder() .size(scrollSize) + .sort(SortBuilders.fieldSort("_doc")) .version(true)); SearchResponse response = client.search(searchRequest, null); try { @@ -241,23 +242,20 @@ public class WatchStore extends AbstractComponent { throw new ElasticsearchException("Partial response while loading watches"); } - if (response.getHits().getTotalHits() > 0) { - response = client.searchScroll(response.getScrollId(), scrollTimeout); - while (response.getHits().hits().length != 0) { - for (SearchHit hit : response.getHits()) { - String id = hit.getId(); - try { - Watch watch = watchParser.parse(id, true, hit.getSourceRef()); - watch.status().version(hit.version()); - watch.version(hit.version()); - watches.put(id, watch); - count++; - } catch (Exception e) { - logger.error("couldn't load watch [{}], ignoring it...", e, id); - } + while (response.getHits().hits().length != 0) { + for (SearchHit hit : response.getHits()) { + String id = hit.getId(); + try { + Watch watch = watchParser.parse(id, true, hit.getSourceRef()); + watch.status().version(hit.version()); + watch.version(hit.version()); + watches.put(id, watch); + count++; + } catch (Exception e) { + logger.error("couldn't load watch [{}], ignoring it...", e, id); } - response = client.searchScroll(response.getScrollId(), scrollTimeout); } + response = client.searchScroll(response.getScrollId(), scrollTimeout); } } finally { client.clearScroll(response.getScrollId()); diff --git a/watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java b/watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java index 2cde65ce491..7a25d03b9b6 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/input/search/SearchInputTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.watcher.input.search; import com.google.common.collect.ImmutableMap; + import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptRequest; import org.elasticsearch.action.search.SearchRequest; @@ -19,6 +20,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.watcher.actions.ActionStatus; @@ -315,24 +317,6 @@ public class SearchInputTests extends ESIntegTestCase { assertThat(names, arrayContaining("test", "test-" + DateTimeFormat.forPattern(dateFormat).print(now.minusDays(1)))); } - @Test(expected = ElasticsearchParseException.class) - public void testParser_ScanNotSupported() throws Exception { - SearchRequest request = client().prepareSearch() - .setSearchType(SearchType.SCAN) - .request() - .source(searchSource() - .query(filteredQuery(matchQuery("event_type", "a"), rangeQuery("_timestamp").from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}")))); - - XContentBuilder builder = jsonBuilder().value(new SearchInput(request, null, null, null)); - XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes()); - parser.nextToken(); - - SearchInputFactory factory = new SearchInputFactory(Settings.EMPTY, ClientProxy.of(client())); - - factory.parseInput("_id", parser); - fail("expected a SearchInputException as search type SCAN should not be supported"); - } - private WatchExecutionContext createContext() { return new TriggeredExecutionContext( new Watch("test-watch", diff --git a/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java b/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java index a1fec34cd81..32ee5d8d37c 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -255,7 +255,7 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase } protected long docCount(String index, String type, SearchSourceBuilder source) { - SearchRequestBuilder builder = client().prepareSearch(index).setSearchType(SearchType.COUNT); + SearchRequestBuilder builder = client().prepareSearch(index).setSize(0); if (type != null) { builder.setTypes(type); } @@ -378,7 +378,7 @@ public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase refresh(); SearchResponse searchResponse = client().prepareSearch(HistoryStore.INDEX_PREFIX + "*") .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .setSearchType(SearchType.COUNT) + .setSize(0) .setQuery(matchQuery("watch_id", watchName)) .get(); return searchResponse.getHits().getTotalHits(); diff --git a/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java b/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java index 908419b3b6c..683588c6506 100644 --- a/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java +++ b/watcher/src/test/java/org/elasticsearch/watcher/transform/search/SearchTransformTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.watcher.transform.search; import com.google.common.collect.ImmutableMap; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.indexedscripts.put.PutIndexedScriptRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -363,27 +362,6 @@ public class SearchTransformTests extends ESIntegTestCase { assertThat(names, arrayContaining("idx", "idx-" + DateTimeFormat.forPattern(dateFormat).print(now.minusDays(3)))); } - @Test(expected = ElasticsearchParseException.class) - public void testParser_ScanNotSupported() throws Exception { - SearchRequest request = client().prepareSearch() - .setSearchType(SearchType.SCAN) - .request() - .source(searchSource() - .query(filteredQuery(matchQuery("event_type", "a"), rangeQuery("_timestamp").from("{{ctx.trigger.scheduled_time}}||-30s").to("{{ctx.trigger.triggered_time}}")))); - - SearchTransform searchTransform = TransformBuilders.searchTransform(request).build(); - XContentBuilder builder = jsonBuilder().value(searchTransform); - XContentParser parser = JsonXContent.jsonXContent.createParser(builder.bytes()); - parser.nextToken(); - - DynamicIndexName.Parser indexNamesParser = new DynamicIndexName.Parser(); - SearchTransformFactory factory = new SearchTransformFactory(Settings.EMPTY, ClientProxy.of(client())); - - factory.parseTransform("_id", parser); - fail("expected a SearchTransformException as search type SCAN should not be supported"); - } - - @Test public void testSearch_InlineTemplate() throws Exception { WatchExecutionContext ctx = createContext();