Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-03-03 16:09:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-9678 update Elasticsearch REST API processor integration-tests for Elasticsearch 8.x

Browse Source 
Signed-off-by: Joe Gresock <jgresock@gmail.com>

This closes #5822.
This commit is contained in:
Chris Sampson 2022-03-01 23:24:35 +00:00 committed by Joe Gresock
parent cf7af1b463
commit 4cc20e6c06
No known key found for this signature in database
GPG Key ID: 37F5B9B6E258C8B7
12 changed files with 147 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java
View File

@ -195,6 +195,14 @@ public interface ElasticSearchClientService extends ControllerService {
*/
UpdateOperationResponse updateByQuery(String query, String index, String type, Map<String, String> requestParameters);
/**
* Refresh index/indices.
*
* @param index The index to target, if omitted then all indices will be updated.
* @param requestParameters A collection of URL request parameters. Optional.
*/
void refresh(final String index, final Map<String, String> requestParameters);
/**
* Get a document by ID.
*

18
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md
View File

@ -43,7 +43,13 @@ mvn -P integration-tests,elasticsearch-oss,elasticsearch-6 clean verify
### Elasticsearch 7
Test integration with Elasticsearch 7.x:
[elasticsearch-oss](https://www.elastic.co/downloads/past-releases#elasticsearch-oss) was discontinued after `7.10.2`,
so the use of `elasticsearch-oss` is unnecessary for newer versions.
For 7.x, we have two separate profiles:
1. `elasticsearch-7` that can be used with `oss` (no X-Pack) and `default` (with X-Pack) flavours
2. `elasticsearch-7-no-oss` that can only be used with the `default` flavour (using a newer version of [elasticsearch](https://www.elastic.co/downloads/past-releases#elasticsearch))
#### With X-Pack
@ -51,6 +57,8 @@ Allows for testing of some X-Pack only features such as "Point in Time" querying
```bash
mvn -P integration-tests,elasticsearch-default,elasticsearch-7 clean verify
sleep 2
mvn -P integration-tests,elasticsearch-default,elasticsearch-7-no-oss clean verify
```
#### Without X-Pack
@ -58,3 +66,11 @@ mvn -P integration-tests,elasticsearch-default,elasticsearch-7 clean verify
```bash
mvn -P integration-tests,elasticsearch-oss,elasticsearch-7 clean verify
```
### Elasticsearch 8
Test integration with Elasticsearch 8.x (with X-Pack):
```bash
mvn -P integration-tests,elasticsearch-default,elasticsearch-8 clean verify
```

48
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml
View File

@ -28,6 +28,7 @@
<properties>
<!-- use with integration-tests only -->
<es.int.version>5.6.16</es.int.version>
<alexcojocaru.plugin.version>6.19</alexcojocaru.plugin.version>
<es.int.script.name>setup-5.script</es.int.script.name>
<es.int.type.name>faketype</es.int.type.name>
<es.int.path.conf>src/test/resources/conf-5/</es.int.path.conf>
@ -214,11 +215,6 @@
<!-- use 3.0.0-M3 due to a classpath/class loader issue in -M5, expected to be fixed in M6+ -->
<version>3.0.0-M3</version>
</plugin>
<plugin>
<groupId>com.github.alexcojocaru</groupId>
<artifactId>elasticsearch-maven-plugin</artifactId>
<version>6.19</version>
</plugin>
</plugins>
</pluginManagement>
</build>
@ -231,7 +227,8 @@
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<es.int.version>6.8.19</es.int.version>
<es.int.version>6.8.23</es.int.version>
<alexcojocaru.plugin.version>6.19</alexcojocaru.plugin.version>
<es.int.type.name>_doc</es.int.type.name>
<es.int.script.name>setup-6.script</es.int.script.name>
<es.int.path.conf />
@ -245,14 +242,43 @@
</activation>
<properties>
<es.int.version>7.10.2</es.int.version>
<alexcojocaru.plugin.version>6.19</alexcojocaru.plugin.version>
<es.int.script.name>setup-7.script</es.int.script.name>
<es.int.type.name />
<es.int.path.conf />
</properties>
</profile>
<profile>
<!-- use with elasticsearch-default -->
<id>elasticsearch-7-no-oss</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<es.int.version>7.17.1</es.int.version>
<alexcojocaru.plugin.version>6.19</alexcojocaru.plugin.version>
<es.int.script.name>setup-7.script</es.int.script.name>
<es.int.type.name />
<es.int.path.conf />
</properties>
</profile>
<profile>
<!-- use with elasticsearch-default -->
<id>elasticsearch-8</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<es.int.version>8.0.1</es.int.version>
<alexcojocaru.plugin.version>6.20</alexcojocaru.plugin.version>
<es.int.script.name>setup-8.script</es.int.script.name>
<es.int.type.name />
<es.int.path.conf />
</properties>
</profile>
<profile>
<!-- OSS Elasticsearch (no XPack features); required for ES 5.x or < 6.3-->
<!-- OSS Elasticsearch (no XPack features); required for ES 5.x or < 6.3 and option for 7.x up to 7.10.2 -->
<id>elasticsearch-oss</id>
<activation>
<activeByDefault>false</activeByDefault>
@ -273,6 +299,7 @@
<plugin>
<groupId>com.github.alexcojocaru</groupId>
<artifactId>elasticsearch-maven-plugin</artifactId>
<version>${alexcojocaru.plugin.version}</version>
<configuration>
<clusterName>${es.int.clusterName}</clusterName>
<transportPort>${es.int.transportPort}</transportPort>
@ -280,7 +307,7 @@
<version>${es.int.version}</version>
<timeout>${es.int.timeout}</timeout>
<logLevel>${es.int.logLevel}</logLevel>
<pathInitScript>${project.basedir}/src/test/resources/${es.int.script.name}</pathInitScript>
<pathInitScript>${project.basedir}/src/test/resources/${es.int.script.name}</pathInitScript>
<keepExistingData>false</keepExistingData>
<pathConf>${es.int.path.conf}</pathConf>
</configuration>
@ -306,7 +333,7 @@
</profile>
<profile>
<!-- Elasticsearch (default) with XPack (only for ES 6.3+ although there are XPack permission problems in 6.x startup) -->
<!-- Elasticsearch (default) with XPack (only for ES 6.3+ & 7.x although there are XPack permission problems in 6.x startup) -->
<id>elasticsearch-default</id>
<activation>
<activeByDefault>false</activeByDefault>
@ -327,6 +354,7 @@
<plugin>
<groupId>com.github.alexcojocaru</groupId>
<artifactId>elasticsearch-maven-plugin</artifactId>
<version>${alexcojocaru.plugin.version}</version>
<configuration>
<flavour>default</flavour>
<clusterName>${es.int.clusterName}</clusterName>
@ -335,7 +363,7 @@
<version>${es.int.version}</version>
<timeout>${es.int.timeout}</timeout>
<logLevel>${es.int.logLevel}</logLevel>
<pathInitScript>${project.basedir}/src/test/resources/${es.int.script.name}</pathInitScript>
<pathInitScript>${project.basedir}/src/test/resources/${es.int.script.name}</pathInitScript>
<keepExistingData>false</keepExistingData>
</configuration>
<executions>

17
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java
View File

@ -389,7 +389,7 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
return new DeleteOperationResponse(watch.getDuration(TimeUnit.MILLISECONDS));
}
@Override
public UpdateOperationResponse updateByQuery(final String query, final String index, final String type, final Map<String, String> requestParameters) {
final long start = System.currentTimeMillis();
final Response response = runQuery("_update_by_query", query, index, type, requestParameters);
@ -401,6 +401,21 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
return new UpdateOperationResponse(end - start);
}
@Override
public void refresh(final String index, final Map<String, String> requestParameters) {
try {
final StringBuilder endpoint = new StringBuilder();
if (StringUtils.isNotBlank(index) && !"/".equals(index)) {
endpoint.append(index);
}
endpoint.append("/_refresh");
final Response response = performRequest("POST", endpoint.toString(), requestParameters, null);
parseResponseWarningHeaders(response);
} catch (final Exception ex) {
throw new ElasticsearchException(ex);
}
}
@SuppressWarnings("unchecked")
@Override
public Map<String, Object> get(final String index, final String type, final String id, final Map<String, String> requestParameters) {

9
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy
View File

@ -58,6 +58,7 @@ class ElasticSearchClientService_IT {
static final ComparableVersion VERSION = new ComparableVersion(System.getProperty("es_version", "0.0.0"))
static final ComparableVersion ES_7_10 = new ComparableVersion("7.10")
static final ComparableVersion ES_8_0 = new ComparableVersion("8.0")
static final String FLAVOUR = System.getProperty("es_flavour")
static final String DEFAULT = "default"
@ -117,6 +118,8 @@ class ElasticSearchClientService_IT {
ex.printStackTrace()
throw ex
}
service.refresh(null, null);
}
@After
@ -223,9 +226,13 @@ class ElasticSearchClientService_IT {
@Test
void testSearchWarnings() {
Assume.assumeTrue("Requires version <8.0 (no search API deprecations yet for 8.x)", VERSION < ES_8_0)
String query
String type = TYPE
if (VERSION.toString().startsWith("7.")) {
if (VERSION.toString().startsWith("8.")) {
// TODO: something that's deprecated when the 8.x branch progresses to include search-API deprecations
} else if (VERSION.toString().startsWith("7.")) {
// querying with _type in ES 7.x is deprecated
query = prettyPrint(toJson([size: 1, query: [match_all: [:]]]))
type = "a-type"

4
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy
View File

@ -68,6 +68,10 @@ class TestElasticSearchClientService extends AbstractControllerService implement
return null
}
@Override
void refresh(final String index, final Map<String, String> requestParameters) {
}
@Override
Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
return data

2
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script
View File

@ -46,5 +46,3 @@ PUT:nested/faketype/1:{"msg":"Hello, world","subField":{"longField":150000,"date
PUT:nested/faketype/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}}
PUT:nested/faketype/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}}
# refresh all indices before testing
POST:_refresh:{}

2
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script
View File

@ -46,5 +46,3 @@ PUT:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateFiel
PUT:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}}
PUT:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}}
# refresh all indices before testing
POST:_refresh:{}

2
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
View File

@ -46,5 +46,3 @@ POST:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateFie
POST:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}}
POST:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}}
# refresh all indices before testing
POST:_refresh:{}

48
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script Normal file
View File

@ -0,0 +1,48 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#create mapping
PUT:user_details/:{ "mappings":{ "properties":{ "email":{"type":"keyword"},"phone":{"type": "keyword"},"accessKey":{"type": "keyword"}}}}
PUT:messages/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
PUT:complex/:{"mappings":{ "properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"}}}}}}
PUT:nested/:{"mappings":{ "properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"},"deeper":{"type":"nested","properties":{"secretz":{"type":"keyword"},"deepest":{"type":"nested","properties":{"super_secret":{"type":"keyword"}}}}}}}}}}
PUT:bulk_a/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
PUT:bulk_b/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
PUT:bulk_c/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
PUT:error_handler:{ "mappings": { "properties": { "msg": { "type": "keyword" }, "intField": { "type": "integer" }}}}
#add document
POST:messages/_doc/1:{ "msg":"one" }
POST:messages/_doc/2:{ "msg":"two" }
POST:messages/_doc/3:{ "msg":"two" }
POST:messages/_doc/4:{ "msg":"three" }
POST:messages/_doc/5:{ "msg":"three" }
POST:messages/_doc/6:{ "msg":"three" }
POST:messages/_doc/7:{ "msg":"four" }
POST:messages/_doc/8:{ "msg":"four" }
POST:messages/_doc/9:{ "msg":"four" }
POST:messages/_doc/10:{ "msg":"four" }
POST:messages/_doc/11:{ "msg":"five" }
POST:messages/_doc/12:{ "msg":"five" }
POST:messages/_doc/13:{ "msg":"five" }
POST:messages/_doc/14:{ "msg":"five" }
POST:messages/_doc/15:{ "msg":"five" }
POST:complex/_doc/1:{"msg":"Hello, world","subField":{"longField":100000,"dateField":"2018-04-10T12:18:05Z"}}
POST:user_details/_doc/1:{ "email": "john.smith@company.com", "phone": "123-456-7890", "accessKey": "ABCDE"}
POST:user_details/_doc/2:{ "email": "jane.doe@company.com", "phone": "098-765-4321", "accessKey": "GHIJK"}
POST:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"No one should see this!","deepest":{"super_secret":"Got nothin to hide"}}}}
POST:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}}
POST:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}}

4
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy
View File

@ -97,6 +97,10 @@ class TestElasticsearchClientService extends AbstractControllerService implement
return new UpdateOperationResponse(100L)
}
@Override
void refresh(final String index, final Map<String, String> requestParameters) {
}
@Override
Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
common(throwErrorInGet || throwNotFoundInGet, requestParameters)

4
nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy
View File

@ -65,6 +65,10 @@ class AbstractMockElasticsearchClient extends AbstractControllerService implemen
return null
}
@Override
void refresh(final String index, final Map<String, String> requestParameters) {
}
@Override
Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
return null