diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java index 138d0a900d..d1dc33a37c 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java @@ -195,6 +195,14 @@ public interface ElasticSearchClientService extends ControllerService { */ UpdateOperationResponse updateByQuery(String query, String index, String type, Map requestParameters); + /** + * Refresh index/indices. + * + * @param index The index to target, if omitted then all indices will be updated. + * @param requestParameters A collection of URL request parameters. Optional. + */ + void refresh(final String index, final Map requestParameters); + /** * Get a document by ID. * diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md index 5ca1e4bbb3..023544d4e9 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md @@ -43,7 +43,13 @@ mvn -P integration-tests,elasticsearch-oss,elasticsearch-6 clean verify ### Elasticsearch 7 -Test integration with Elasticsearch 7.x: +[elasticsearch-oss](https://www.elastic.co/downloads/past-releases#elasticsearch-oss) was discontinued after `7.10.2`, +so the use of `elasticsearch-oss` is unnecessary for newer versions. + +For 7.x, we have two separate profiles: + +1. `elasticsearch-7` that can be used with `oss` (no X-Pack) and `default` (with X-Pack) flavours +2. `elasticsearch-7-no-oss` that can only be used with the `default` flavour (using a newer version of [elasticsearch](https://www.elastic.co/downloads/past-releases#elasticsearch)) #### With X-Pack @@ -51,6 +57,8 @@ Allows for testing of some X-Pack only features such as "Point in Time" querying ```bash mvn -P integration-tests,elasticsearch-default,elasticsearch-7 clean verify +sleep 2 +mvn -P integration-tests,elasticsearch-default,elasticsearch-7-no-oss clean verify ``` #### Without X-Pack @@ -58,3 +66,11 @@ mvn -P integration-tests,elasticsearch-default,elasticsearch-7 clean verify ```bash mvn -P integration-tests,elasticsearch-oss,elasticsearch-7 clean verify ``` + +### Elasticsearch 8 + +Test integration with Elasticsearch 8.x (with X-Pack): + +```bash +mvn -P integration-tests,elasticsearch-default,elasticsearch-8 clean verify +``` diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml index 0f97fa9ef7..050e2adfef 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml @@ -28,6 +28,7 @@ 5.6.16 + 6.19 setup-5.script faketype src/test/resources/conf-5/ @@ -214,11 +215,6 @@ 3.0.0-M3 - - com.github.alexcojocaru - elasticsearch-maven-plugin - 6.19 - @@ -231,7 +227,8 @@ false - 6.8.19 + 6.8.23 + 6.19 _doc setup-6.script @@ -245,14 +242,43 @@ 7.10.2 + 6.19 setup-7.script + + + elasticsearch-7-no-oss + + false + + + 7.17.1 + 6.19 + setup-7.script + + + + + + + elasticsearch-8 + + false + + + 8.0.1 + 6.20 + setup-8.script + + + + - + elasticsearch-oss false @@ -273,6 +299,7 @@ com.github.alexcojocaru elasticsearch-maven-plugin + ${alexcojocaru.plugin.version} ${es.int.clusterName} ${es.int.transportPort} @@ -280,7 +307,7 @@ ${es.int.version} ${es.int.timeout} ${es.int.logLevel} - ${project.basedir}/src/test/resources/${es.int.script.name} + ${project.basedir}/src/test/resources/${es.int.script.name} false ${es.int.path.conf} @@ -306,7 +333,7 @@ - + elasticsearch-default false @@ -327,6 +354,7 @@ com.github.alexcojocaru elasticsearch-maven-plugin + ${alexcojocaru.plugin.version} default ${es.int.clusterName} @@ -335,7 +363,7 @@ ${es.int.version} ${es.int.timeout} ${es.int.logLevel} - ${project.basedir}/src/test/resources/${es.int.script.name} + ${project.basedir}/src/test/resources/${es.int.script.name} false diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java index 202c8737b1..58dc99ea52 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java @@ -389,7 +389,7 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im return new DeleteOperationResponse(watch.getDuration(TimeUnit.MILLISECONDS)); } - + @Override public UpdateOperationResponse updateByQuery(final String query, final String index, final String type, final Map requestParameters) { final long start = System.currentTimeMillis(); final Response response = runQuery("_update_by_query", query, index, type, requestParameters); @@ -401,6 +401,21 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im return new UpdateOperationResponse(end - start); } + @Override + public void refresh(final String index, final Map requestParameters) { + try { + final StringBuilder endpoint = new StringBuilder(); + if (StringUtils.isNotBlank(index) && !"/".equals(index)) { + endpoint.append(index); + } + endpoint.append("/_refresh"); + final Response response = performRequest("POST", endpoint.toString(), requestParameters, null); + parseResponseWarningHeaders(response); + } catch (final Exception ex) { + throw new ElasticsearchException(ex); + } + } + @SuppressWarnings("unchecked") @Override public Map get(final String index, final String type, final String id, final Map requestParameters) { diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy index aefc38a3c3..5c77cc1010 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy @@ -58,6 +58,7 @@ class ElasticSearchClientService_IT { static final ComparableVersion VERSION = new ComparableVersion(System.getProperty("es_version", "0.0.0")) static final ComparableVersion ES_7_10 = new ComparableVersion("7.10") + static final ComparableVersion ES_8_0 = new ComparableVersion("8.0") static final String FLAVOUR = System.getProperty("es_flavour") static final String DEFAULT = "default" @@ -117,6 +118,8 @@ class ElasticSearchClientService_IT { ex.printStackTrace() throw ex } + + service.refresh(null, null); } @After @@ -223,9 +226,13 @@ class ElasticSearchClientService_IT { @Test void testSearchWarnings() { + Assume.assumeTrue("Requires version <8.0 (no search API deprecations yet for 8.x)", VERSION < ES_8_0) + String query String type = TYPE - if (VERSION.toString().startsWith("7.")) { + if (VERSION.toString().startsWith("8.")) { + // TODO: something that's deprecated when the 8.x branch progresses to include search-API deprecations + } else if (VERSION.toString().startsWith("7.")) { // querying with _type in ES 7.x is deprecated query = prettyPrint(toJson([size: 1, query: [match_all: [:]]])) type = "a-type" diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy index ffb3e39a3b..4e0f42546b 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy @@ -68,6 +68,10 @@ class TestElasticSearchClientService extends AbstractControllerService implement return null } + @Override + void refresh(final String index, final Map requestParameters) { + } + @Override Map get(String index, String type, String id, Map requestParameters) { return data diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script index 429a74ddc9..c2cffc483c 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script @@ -46,5 +46,3 @@ PUT:nested/faketype/1:{"msg":"Hello, world","subField":{"longField":150000,"date PUT:nested/faketype/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}} PUT:nested/faketype/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}} -# refresh all indices before testing -POST:_refresh:{} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script index 739c0f2cab..c3522f5b87 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script @@ -46,5 +46,3 @@ PUT:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateFiel PUT:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}} PUT:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}} -# refresh all indices before testing -POST:_refresh:{} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script index 4328390abc..d7cfcc9de1 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script @@ -46,5 +46,3 @@ POST:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateFie POST:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}} POST:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}} -# refresh all indices before testing -POST:_refresh:{} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script new file mode 100644 index 0000000000..d7cfcc9de1 --- /dev/null +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#create mapping +PUT:user_details/:{ "mappings":{ "properties":{ "email":{"type":"keyword"},"phone":{"type": "keyword"},"accessKey":{"type": "keyword"}}}} +PUT:messages/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}} +PUT:complex/:{"mappings":{ "properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"}}}}}} +PUT:nested/:{"mappings":{ "properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"},"deeper":{"type":"nested","properties":{"secretz":{"type":"keyword"},"deepest":{"type":"nested","properties":{"super_secret":{"type":"keyword"}}}}}}}}}} +PUT:bulk_a/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}} +PUT:bulk_b/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}} +PUT:bulk_c/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}} +PUT:error_handler:{ "mappings": { "properties": { "msg": { "type": "keyword" }, "intField": { "type": "integer" }}}} + +#add document +POST:messages/_doc/1:{ "msg":"one" } +POST:messages/_doc/2:{ "msg":"two" } +POST:messages/_doc/3:{ "msg":"two" } +POST:messages/_doc/4:{ "msg":"three" } +POST:messages/_doc/5:{ "msg":"three" } +POST:messages/_doc/6:{ "msg":"three" } +POST:messages/_doc/7:{ "msg":"four" } +POST:messages/_doc/8:{ "msg":"four" } +POST:messages/_doc/9:{ "msg":"four" } +POST:messages/_doc/10:{ "msg":"four" } +POST:messages/_doc/11:{ "msg":"five" } +POST:messages/_doc/12:{ "msg":"five" } +POST:messages/_doc/13:{ "msg":"five" } +POST:messages/_doc/14:{ "msg":"five" } +POST:messages/_doc/15:{ "msg":"five" } +POST:complex/_doc/1:{"msg":"Hello, world","subField":{"longField":100000,"dateField":"2018-04-10T12:18:05Z"}} +POST:user_details/_doc/1:{ "email": "john.smith@company.com", "phone": "123-456-7890", "accessKey": "ABCDE"} +POST:user_details/_doc/2:{ "email": "jane.doe@company.com", "phone": "098-765-4321", "accessKey": "GHIJK"} +POST:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"No one should see this!","deepest":{"super_secret":"Got nothin to hide"}}}} +POST:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}} +POST:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}} + diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy index 74103806ac..02eed951f7 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy @@ -97,6 +97,10 @@ class TestElasticsearchClientService extends AbstractControllerService implement return new UpdateOperationResponse(100L) } + @Override + void refresh(final String index, final Map requestParameters) { + } + @Override Map get(String index, String type, String id, Map requestParameters) { common(throwErrorInGet || throwNotFoundInGet, requestParameters) diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy index dca1f853ac..52a0316ab6 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy @@ -65,6 +65,10 @@ class AbstractMockElasticsearchClient extends AbstractControllerService implemen return null } + @Override + void refresh(final String index, final Map requestParameters) { + } + @Override Map get(String index, String type, String id, Map requestParameters) { return null