honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add `ignore_missing` option to SplitProcessor (#20982) 

Closes #20840.
This commit is contained in:
Tal Levy 2016-11-16 15:46:09 +02:00 committed by GitHub
parent 04b712bdc5
commit 6796464f16
8 changed files with 73 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
View File

@ -116,6 +116,28 @@ public final class IngestDocument {
return cast(path, context, clazz);
}
/**
* Returns the value contained in the document for the provided path
*
* @param path The path within the document in dot-notation
* @param clazz The expected class of the field value
* @param ignoreMissing The flag to determine whether to throw an exception when `path` is not found in the document.
* @return the value for the provided path if existing, null otherwise.
* @throws IllegalArgumentException only if ignoreMissing is false and the path is null, empty, invalid, if the field doesn't exist
* or if the field that is found at the provided path is not of the expected type.
*/
public <T> T getFieldValue(String path, Class<T> clazz, boolean ignoreMissing) {
try {
return getFieldValue(path, clazz);
} catch (IllegalArgumentException e) {
if (ignoreMissing && hasField(path) != true) {
return null;
} else {
throw e;
}
}
}
/**
* Returns the value contained in the document with the provided templated path
* @param pathTemplate The path within the document in dot-notation

1
docs/reference/ingest/ingest-node.asciidoc
View File

@ -1630,6 +1630,7 @@ Splits a field into an array using a separator character. Only works on string f
| Name | Required | Default | Description
| `field` | yes | - | The field to split
| `separator` | yes | - | A regex which matches the separator, eg `,` or `\s+`
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
|======
[source,js]

14
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
View File

@ -34,7 +34,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
private final String field;
private final boolean ignoreMissing;
protected AbstractStringProcessor(String tag, String field, boolean ignoreMissing) {
AbstractStringProcessor(String tag, String field, boolean ignoreMissing) {
super(tag);
this.field = field;
this.ignoreMissing = ignoreMissing;
@ -50,16 +50,8 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
@Override
public final void execute(IngestDocument document) {
String val;
String val = document.getFieldValue(field, String.class, ignoreMissing);
try {
val = document.getFieldValue(field, String.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing && document.hasField(field) != true) {
return;
}
throw e;
}
if (val == null && ignoreMissing) {
return;
} else if (val == null) {
@ -72,7 +64,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
protected abstract String process(String value);
abstract static class Factory implements Processor.Factory {
protected final String processorType;
final String processorType;
protected Factory(String processorType) {
this.processorType = processorType;

11
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
View File

@ -142,18 +142,9 @@ public final class ConvertProcessor extends AbstractProcessor {
@Override
public void execute(IngestDocument document) {
Object oldValue = null;
Object oldValue = document.getFieldValue(field, Object.class, ignoreMissing);
Object newValue;
try {
oldValue = document.getFieldValue(field, Object.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing) {
return;
}
throw e;
}
if (oldValue == null && ignoreMissing) {
return;
} else if (oldValue == null) {

11
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
View File

@ -54,16 +54,7 @@ public final class GrokProcessor extends AbstractProcessor {
@Override
public void execute(IngestDocument ingestDocument) throws Exception {
String fieldValue;
try {
fieldValue = ingestDocument.getFieldValue(matchField, String.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing && ingestDocument.hasField(matchField) != true) {
return;
}
throw e;
}
String fieldValue = ingestDocument.getFieldValue(matchField, String.class, ignoreMissing);
if (fieldValue == null && ignoreMissing) {
return;

20
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
View File

@ -40,11 +40,13 @@ public final class SplitProcessor extends AbstractProcessor {
private final String field;
private final String separator;
private final boolean ignoreMissing;
SplitProcessor(String tag, String field, String separator) {
SplitProcessor(String tag, String field, String separator, boolean ignoreMissing) {
super(tag);
this.field = field;
this.separator = separator;
this.ignoreMissing = ignoreMissing;
}
String getField() {
@ -55,12 +57,20 @@ public final class SplitProcessor extends AbstractProcessor {
return separator;
}
boolean isIgnoreMissing() {
return ignoreMissing;
}
@Override
public void execute(IngestDocument document) {
String oldVal = document.getFieldValue(field, String.class);
if (oldVal == null) {
String oldVal = document.getFieldValue(field, String.class, ignoreMissing);
if (oldVal == null && ignoreMissing) {
return;
} else if (oldVal == null) {
throw new IllegalArgumentException("field [" + field + "] is null, cannot split.");
}
String[] strings = oldVal.split(separator);
List<String> splitList = new ArrayList<>(strings.length);
Collections.addAll(splitList, strings);
@ -77,7 +87,9 @@ public final class SplitProcessor extends AbstractProcessor {
public SplitProcessor create(Map<String, Processor.Factory> registry, String processorTag,
Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
return new SplitProcessor(processorTag, field, ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"));
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new SplitProcessor(processorTag, field,
ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"), ignoreMissing);
}
}
}

1
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
View File

@ -39,6 +39,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
assertThat(splitProcessor.getTag(), equalTo(processorTag));
assertThat(splitProcessor.getField(), equalTo("field1"));
assertThat(splitProcessor.getSeparator(), equalTo("\\."));
assertFalse(splitProcessor.isIgnoreMissing());
}
public void testCreateNoFieldPresent() throws Exception {

30
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
View File

@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@ -38,7 +39,7 @@ public class SplitProcessorTests extends ESTestCase {
public void testSplit() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1")));
}
@ -46,7 +47,7 @@ public class SplitProcessorTests extends ESTestCase {
public void testSplitFieldNotFound() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false);
try {
processor.execute(ingestDocument);
fail("split processor should have failed");
@ -56,8 +57,9 @@ public class SplitProcessorTests extends ESTestCase {
}
public void testSplitNullValue() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(),
Collections.singletonMap("field", null));
Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.", false);
try {
processor.execute(ingestDocument);
fail("split processor should have failed");
@ -66,11 +68,29 @@ public class SplitProcessorTests extends ESTestCase {
}
}
public void testSplitNullValueWithIgnoreMissing() throws Exception {
String fieldName = RandomDocumentPicks.randomFieldName(random());
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
Collections.singletonMap(fieldName, null));
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testSplitNonExistentWithIgnoreMissing() throws Exception {
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.", true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testSplitNonStringValue() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
ingestDocument.setFieldValue(fieldName, randomInt());
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false);
try {
processor.execute(ingestDocument);
fail("split processor should have failed");