diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 8010c6e6c3f..670ff19fdab 100644 --- a/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -116,6 +116,28 @@ public final class IngestDocument { return cast(path, context, clazz); } + /** + * Returns the value contained in the document for the provided path + * + * @param path The path within the document in dot-notation + * @param clazz The expected class of the field value + * @param ignoreMissing The flag to determine whether to throw an exception when `path` is not found in the document. + * @return the value for the provided path if existing, null otherwise. + * @throws IllegalArgumentException only if ignoreMissing is false and the path is null, empty, invalid, if the field doesn't exist + * or if the field that is found at the provided path is not of the expected type. + */ + public T getFieldValue(String path, Class clazz, boolean ignoreMissing) { + try { + return getFieldValue(path, clazz); + } catch (IllegalArgumentException e) { + if (ignoreMissing && hasField(path) != true) { + return null; + } else { + throw e; + } + } + } + /** * Returns the value contained in the document with the provided templated path * @param pathTemplate The path within the document in dot-notation diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 1ac1134f779..ed50a0f717b 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -1627,9 +1627,10 @@ Splits a field into an array using a separator character. Only works on string f .Split Options [options="header"] |====== -| Name | Required | Default | Description -| `field` | yes | - | The field to split -| `separator` | yes | - | A regex which matches the separator, eg `,` or `\s+` +| Name | Required | Default | Description +| `field` | yes | - | The field to split +| `separator` | yes | - | A regex which matches the separator, eg `,` or `\s+` +| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document |====== [source,js] diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java index 9e157dc994b..477ec5605b0 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java @@ -34,7 +34,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor { private final String field; private final boolean ignoreMissing; - protected AbstractStringProcessor(String tag, String field, boolean ignoreMissing) { + AbstractStringProcessor(String tag, String field, boolean ignoreMissing) { super(tag); this.field = field; this.ignoreMissing = ignoreMissing; @@ -50,16 +50,8 @@ abstract class AbstractStringProcessor extends AbstractProcessor { @Override public final void execute(IngestDocument document) { - String val; + String val = document.getFieldValue(field, String.class, ignoreMissing); - try { - val = document.getFieldValue(field, String.class); - } catch (IllegalArgumentException e) { - if (ignoreMissing && document.hasField(field) != true) { - return; - } - throw e; - } if (val == null && ignoreMissing) { return; } else if (val == null) { @@ -72,7 +64,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor { protected abstract String process(String value); abstract static class Factory implements Processor.Factory { - protected final String processorType; + final String processorType; protected Factory(String processorType) { this.processorType = processorType; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index c57d93fa9c6..3cf35c76117 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -142,18 +142,9 @@ public final class ConvertProcessor extends AbstractProcessor { @Override public void execute(IngestDocument document) { - Object oldValue = null; + Object oldValue = document.getFieldValue(field, Object.class, ignoreMissing); Object newValue; - try { - oldValue = document.getFieldValue(field, Object.class); - } catch (IllegalArgumentException e) { - if (ignoreMissing) { - return; - } - throw e; - } - if (oldValue == null && ignoreMissing) { return; } else if (oldValue == null) { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index 22992a04076..f271da198f8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -54,16 +54,7 @@ public final class GrokProcessor extends AbstractProcessor { @Override public void execute(IngestDocument ingestDocument) throws Exception { - String fieldValue; - - try { - fieldValue = ingestDocument.getFieldValue(matchField, String.class); - } catch (IllegalArgumentException e) { - if (ignoreMissing && ingestDocument.hasField(matchField) != true) { - return; - } - throw e; - } + String fieldValue = ingestDocument.getFieldValue(matchField, String.class, ignoreMissing); if (fieldValue == null && ignoreMissing) { return; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java index 8ff841ef0e4..eadbed4bcc6 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java @@ -40,11 +40,13 @@ public final class SplitProcessor extends AbstractProcessor { private final String field; private final String separator; + private final boolean ignoreMissing; - SplitProcessor(String tag, String field, String separator) { + SplitProcessor(String tag, String field, String separator, boolean ignoreMissing) { super(tag); this.field = field; this.separator = separator; + this.ignoreMissing = ignoreMissing; } String getField() { @@ -55,12 +57,20 @@ public final class SplitProcessor extends AbstractProcessor { return separator; } + boolean isIgnoreMissing() { + return ignoreMissing; + } + @Override public void execute(IngestDocument document) { - String oldVal = document.getFieldValue(field, String.class); - if (oldVal == null) { + String oldVal = document.getFieldValue(field, String.class, ignoreMissing); + + if (oldVal == null && ignoreMissing) { + return; + } else if (oldVal == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot split."); } + String[] strings = oldVal.split(separator); List splitList = new ArrayList<>(strings.length); Collections.addAll(splitList, strings); @@ -77,7 +87,9 @@ public final class SplitProcessor extends AbstractProcessor { public SplitProcessor create(Map registry, String processorTag, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); - return new SplitProcessor(processorTag, field, ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator")); + boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); + return new SplitProcessor(processorTag, field, + ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"), ignoreMissing); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java index bef9a84a1e2..68580eb8f1d 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java @@ -39,6 +39,7 @@ public class SplitProcessorFactoryTests extends ESTestCase { assertThat(splitProcessor.getTag(), equalTo(processorTag)); assertThat(splitProcessor.getField(), equalTo("field1")); assertThat(splitProcessor.getSeparator(), equalTo("\\.")); + assertFalse(splitProcessor.isIgnoreMissing()); } public void testCreateNoFieldPresent() throws Exception { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java index edb3639679c..253ddfd194b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -38,7 +39,7 @@ public class SplitProcessorTests extends ESTestCase { public void testSplit() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1"); - Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\."); + Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1"))); } @@ -46,7 +47,7 @@ public class SplitProcessorTests extends ESTestCase { public void testSplitFieldNotFound() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\."); + Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false); try { processor.execute(ingestDocument); fail("split processor should have failed"); @@ -56,8 +57,9 @@ public class SplitProcessorTests extends ESTestCase { } public void testSplitNullValue() throws Exception { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); - Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\."); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), + Collections.singletonMap("field", null)); + Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.", false); try { processor.execute(ingestDocument); fail("split processor should have failed"); @@ -66,11 +68,29 @@ public class SplitProcessorTests extends ESTestCase { } } + public void testSplitNullValueWithIgnoreMissing() throws Exception { + String fieldName = RandomDocumentPicks.randomFieldName(random()); + IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), + Collections.singletonMap(fieldName, null)); + IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); + Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", true); + processor.execute(ingestDocument); + assertIngestDocument(originalIngestDocument, ingestDocument); + } + + public void testSplitNonExistentWithIgnoreMissing() throws Exception { + IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); + IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); + Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.", true); + processor.execute(ingestDocument); + assertIngestDocument(originalIngestDocument, ingestDocument); + } + public void testSplitNonStringValue() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); ingestDocument.setFieldValue(fieldName, randomInt()); - Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\."); + Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false); try { processor.execute(ingestDocument); fail("split processor should have failed");