From d59a07245ff08117b3b4245dbba84b0b093c68a8 Mon Sep 17 00:00:00 2001 From: Wietze Date: Wed, 8 Aug 2018 17:17:47 +0100 Subject: [PATCH] NIFI-5500 Adding support for array fields when outputting to attributes when using QueryElasticsearchHttp NIFI-5500 Optimised array-to-string conversion This closes #2942 Signed-off-by: Mike Thomsen --- .../elasticsearch/QueryElasticsearchHttp.java | 13 ++++++++++++- .../elasticsearch/TestQueryElasticsearchHttp.java | 1 + .../src/test/resources/query-page1.json | 10 ++++++---- .../src/test/resources/query-page2.json | 5 +++-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java index 33eac3bc2c..67deb87c9c 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/QueryElasticsearchHttp.java @@ -432,7 +432,18 @@ public class QueryElasticsearchHttp extends AbstractElasticsearchHttpProcessor { Map attributes = new HashMap<>(); for(Iterator> it = source.fields(); it.hasNext(); ) { Entry entry = it.next(); - attributes.put(ATTRIBUTE_PREFIX + entry.getKey(), entry.getValue().asText()); + + String textValue = ""; + if(entry.getValue().isArray()){ + ArrayList text_values = new ArrayList(); + for(Iterator items = entry.getValue().iterator(); items.hasNext(); ) { + text_values.add(items.next().asText()); + } + textValue = StringUtils.join(text_values, ','); + } else { + textValue = entry.getValue().asText(); + } + attributes.put(ATTRIBUTE_PREFIX + entry.getKey(), textValue); } documentFlowFile = session.putAllAttributes(documentFlowFile, attributes); } diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestQueryElasticsearchHttp.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestQueryElasticsearchHttp.java index 50b950ff45..b67aebfff3 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestQueryElasticsearchHttp.java +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestQueryElasticsearchHttp.java @@ -141,6 +141,7 @@ public class TestQueryElasticsearchHttp { QueryElasticsearchHttp.REL_SUCCESS).get(0); assertNotNull(out); assertEquals("blah", new String(out.toByteArray())); + assertEquals("arrays,are,supported,too", out.getAttribute("es.result.tags")); assertEquals("Twitter", out.getAttribute("es.result.source")); } diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/resources/query-page1.json b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/resources/query-page1.json index 50930761f8..649b480ffc 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/resources/query-page1.json +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/resources/query-page1.json @@ -25,7 +25,8 @@ "transit_uri": "file://cluster2/data/outgoing/S3/abc-97b.zip", "object_type": "Provenance Record", "version": "ASVsZu_vShwtGCJpGOObmuSqUJRUC3L_", - "file_size": "3645525" + "file_size": "3645525", + "tags": ["arrays", "are", "supported", "too"] }, "sort": [ 1469198828102 @@ -45,13 +46,14 @@ "transit_uri": "file://cluster2/data/outgoing/S3/abc-a78.zip", "object_type": "Provenance Record", "version": "SjJkrwnv6edIRqJChEYzrE7PeT1hzioz", - "file_size": "4480294" + "file_size": "4480294", + "tags": ["arrays", "are", "supported", "too"] }, "sort": [ 1469198828101 ] - + } ] } - } \ No newline at end of file + } diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/resources/query-page2.json b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/resources/query-page2.json index 1ea27b33f4..01357032be 100644 --- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/resources/query-page2.json +++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/resources/query-page2.json @@ -25,7 +25,8 @@ "transit_uri": "file://cluster2/data/outgoing/S3/abc-42a.zip", "object_type": "Provenance Record", "version": "ArPsIlGBKqDvfL6qQZOVpmDwUEB.nynh", - "file_size": "18206872" + "file_size": "18206872", + "tags": ["arrays", "are", "supported", "too"] }, "sort": [ 1469198828101 @@ -33,4 +34,4 @@ } ] } - } \ No newline at end of file + }