From e3fd1e6c7d304927ccd19f57e758c8601604690c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 8 Aug 2019 10:17:20 +0200 Subject: [PATCH] Add support for overwrite parameter in the enrich processor. (#45029) Similar to how it is supported in the set processor: https://www.elastic.co/guide/en/elasticsearch/reference/current/set-processor.html Relates to #32789 --- .../xpack/enrich/EnrichProcessorFactory.java | 3 +- .../xpack/enrich/ExactMatchProcessor.java | 13 +++- .../enrich/EnrichProcessorFactoryTests.java | 10 +++ .../enrich/ExactMatchProcessorTests.java | 72 +++++++++++++++++-- 4 files changed, 91 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index 1fcceb42afa..82089bc31a5 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -37,6 +37,7 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer specifications; final List> specificationConfig = ConfigurationUtils.readList(TYPE, tag, config, "enrich_values"); @@ -54,7 +55,7 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer specifications; ExactMatchProcessor(String tag, @@ -39,6 +40,7 @@ final class ExactMatchProcessor extends AbstractProcessor { String policyName, String enrichKey, boolean ignoreMissing, + boolean overrideEnabled, List specifications) { this( tag, @@ -46,6 +48,7 @@ final class ExactMatchProcessor extends AbstractProcessor { policyName, enrichKey, ignoreMissing, + overrideEnabled, specifications ); } @@ -55,12 +58,14 @@ final class ExactMatchProcessor extends AbstractProcessor { String policyName, String enrichKey, boolean ignoreMissing, + boolean overrideEnabled, List specifications) { super(tag); this.searchRunner = searchRunner; this.policyName = policyName; this.enrichKey = enrichKey; this.ignoreMissing = ignoreMissing; + this.overrideEnabled = overrideEnabled; this.specifications = specifications; } @@ -111,7 +116,9 @@ final class ExactMatchProcessor extends AbstractProcessor { assert enrichDocument != null : "enrich document for id [" + enrichKey + "] was empty despite non-zero search hits length"; for (EnrichSpecification specification : specifications) { Object enrichFieldValue = enrichDocument.get(specification.sourceField); - ingestDocument.setFieldValue(specification.targetField, enrichFieldValue); + if (overrideEnabled || ingestDocument.hasField(specification.targetField) == false) { + ingestDocument.setFieldValue(specification.targetField, enrichFieldValue); + } } handler.accept(ingestDocument, null); }); @@ -142,6 +149,10 @@ final class ExactMatchProcessor extends AbstractProcessor { return ignoreMissing; } + boolean isOverrideEnabled() { + return overrideEnabled; + } + List getSpecifications() { return specifications; } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index c92dee4710a..665fa8b669a 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -39,6 +39,11 @@ public class EnrichProcessorFactoryTests extends ESTestCase { config.put("ignore_missing", keyIgnoreMissing); } + Boolean overrideEnabled = randomBoolean() ? null : randomBoolean(); + if (overrideEnabled != null) { + config.put("override", overrideEnabled); + } + int numRandomValues = randomIntBetween(1, 8); List> randomValues = new ArrayList<>(numRandomValues); for (int i = 0; i < numRandomValues; i++) { @@ -59,6 +64,11 @@ public class EnrichProcessorFactoryTests extends ESTestCase { assertThat(result.getPolicyName(), equalTo("majestic")); assertThat(result.getEnrichKey(), equalTo("host")); assertThat(result.isIgnoreMissing(), is(keyIgnoreMissing)); + if (overrideEnabled != null) { + assertThat(result.isOverrideEnabled(), is(overrideEnabled)); + } else { + assertThat(result.isOverrideEnabled(), is(true)); + } assertThat(result.getSpecifications().size(), equalTo(numRandomValues)); for (int i = 0; i < numRandomValues; i++) { EnrichSpecification actual = result.getSpecifications().get(i); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java index 2ea93164b06..32b59b2bdfc 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java @@ -56,7 +56,7 @@ public class ExactMatchProcessorTests extends ESTestCase { documents.put("elastic.co", document1); } MockSearchFunction mockSearch = mockedSearchFunction(documents); - ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, true, Arrays.asList(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Collections.singletonMap("domain", "elastic.co")); @@ -85,7 +85,7 @@ public class ExactMatchProcessorTests extends ESTestCase { public void testNoMatch() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(); - ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, true, Arrays.asList(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Collections.singletonMap("domain", "elastic.com")); @@ -115,7 +115,7 @@ public class ExactMatchProcessorTests extends ESTestCase { public void testSearchFailure() throws Exception { String indexName = ".enrich-_name"; MockSearchFunction mockSearch = mockedSearchFunction(new IndexNotFoundException(indexName)); - ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, true, Arrays.asList(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Collections.singletonMap("domain", "elastic.com")); @@ -150,7 +150,7 @@ public class ExactMatchProcessorTests extends ESTestCase { public void testIgnoreKeyMissing() throws Exception { { ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockedSearchFunction(), "_name", "domain", - true, Arrays.asList(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); + true, true, Arrays.asList(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Collections.emptyMap()); @@ -162,7 +162,7 @@ public class ExactMatchProcessorTests extends ESTestCase { } { ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockedSearchFunction(), "_name", "domain", - false, Arrays.asList(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); + false, true, Arrays.asList(new EnrichSpecification("tldRank", "tld_rank"), new EnrichSpecification("tld", "tld"))); IngestDocument ingestDocument = new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, Collections.emptyMap()); IngestDocument[] resultHolder = new IngestDocument[1]; @@ -177,6 +177,43 @@ public class ExactMatchProcessorTests extends ESTestCase { } } + public void testExistingFieldWithOverrideDisabled() throws Exception { + MockSearchFunction mockSearch = mockedSearchFunction(mapOf("elastic.co", mapOf("globalRank", 451, "tldRank", 23, "tld", "co"))); + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, false, + Collections.singletonList(new EnrichSpecification("tld", "tld"))); + + IngestDocument ingestDocument = new IngestDocument(new HashMap<>(mapOf("domain", "elastic.co", "tld", "tld")), mapOf()); + IngestDocument[] resultHolder = new IngestDocument[1]; + Exception[] exceptionHolder = new Exception[1]; + processor.execute(ingestDocument, (result, e) -> { + resultHolder[0] = result; + exceptionHolder[0] = e; + }); + assertThat(exceptionHolder[0], nullValue()); + assertThat(resultHolder[0].hasField("tld"), equalTo(true)); + assertThat(resultHolder[0].getFieldValue("tld", Object.class), equalTo("tld")); + } + + public void testExistingNullFieldWithOverrideDisabled() throws Exception { + MockSearchFunction mockSearch = mockedSearchFunction(mapOf("elastic.co", mapOf("globalRank", 451, "tldRank", 23, "tld", "co"))); + ExactMatchProcessor processor = new ExactMatchProcessor("_tag", mockSearch, "_name", "domain", false, false, + Collections.singletonList(new EnrichSpecification("tld", "tld"))); + + Map source = new HashMap<>(); + source.put("domain", "elastic.co"); + source.put("tld", null); + IngestDocument ingestDocument = new IngestDocument(source, mapOf()); + IngestDocument[] resultHolder = new IngestDocument[1]; + Exception[] exceptionHolder = new Exception[1]; + processor.execute(ingestDocument, (result, e) -> { + resultHolder[0] = result; + exceptionHolder[0] = e; + }); + assertThat(exceptionHolder[0], nullValue()); + assertThat(resultHolder[0].hasField("tld"), equalTo(true)); + assertThat(resultHolder[0].getFieldValue("tld", Object.class), equalTo(null)); + } + private static final class MockSearchFunction implements BiConsumer> { private final SearchResponse mockResponse; private final SetOnce capturedRequest; @@ -240,4 +277,29 @@ public class ExactMatchProcessorTests extends ESTestCase { new Aggregations(Collections.emptyList()), new Suggest(Collections.emptyList()), false, false, null, 1), null, 1, 1, 0, 1, ShardSearchFailure.EMPTY_ARRAY, new SearchResponse.Clusters(1, 1, 0)); } + + static Map mapOf() { + return Collections.emptyMap(); + } + + static Map mapOf(K key1, V value1) { + Map map = new HashMap<>(); + map.put(key1, value1); + return map; + } + + static Map mapOf(K key1, V value1, K key2, V value2) { + Map map = new HashMap<>(); + map.put(key1, value1); + map.put(key2, value2); + return map; + } + + static Map mapOf(String key1, Object value1, String key2, Object value2, String key3, Object value3) { + Map map = new HashMap<>(); + map.put(key1, value1); + map.put(key2, value2); + map.put(key3, value3); + return map; + } }