honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Ingest: Add ignore_missing option to RemoveProc (#31693) 

Added `ignore_missing` setting to the RemoveProcessor to fix #23086
This commit is contained in:
Armin Braun 2018-07-09 10:24:34 +02:00 committed by GitHub
parent c5c860dadb
commit e46ed73379
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 35 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/reference/ingest/ingest-node.asciidoc
View File

@ -1767,8 +1767,9 @@ Removes existing fields. If one field doesn't exist, an exception will be thrown
.Remove Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | Fields to be removed
| Name | Required | Default | Description
| `field` | yes | - | Fields to be removed
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======
Here is an example to remove a single field:

18
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
View File

@ -39,10 +39,12 @@ public final class RemoveProcessor extends AbstractProcessor {
public static final String TYPE = "remove";
private final List<TemplateScript.Factory> fields;
private final boolean ignoreMissing;
RemoveProcessor(String tag, List<TemplateScript.Factory> fields) {
RemoveProcessor(String tag, List<TemplateScript.Factory> fields, boolean ignoreMissing) {
super(tag);
this.fields = new ArrayList<>(fields);
this.ignoreMissing = ignoreMissing;
}
public List<TemplateScript.Factory> getFields() {
@ -51,7 +53,16 @@ public final class RemoveProcessor extends AbstractProcessor {
@Override
public void execute(IngestDocument document) {
fields.forEach(document::removeField);
if (ignoreMissing) {
fields.forEach(field -> {
String path = document.renderTemplate(field);
if (document.hasField(path)) {
document.removeField(path);
}
});
} else {
fields.forEach(document::removeField);
}
}
@Override
@ -83,7 +94,8 @@ public final class RemoveProcessor extends AbstractProcessor {
final List<TemplateScript.Factory> compiledTemplates = fields.stream()
.map(f -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", f, scriptService))
.collect(Collectors.toList());
return new RemoveProcessor(processorTag, compiledTemplates);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new RemoveProcessor(processorTag, compiledTemplates, ignoreMissing);
}
}
}

20
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.test.ESTestCase;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@ -37,7 +38,7 @@ public class RemoveProcessorTests extends ESTestCase {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
Processor processor = new RemoveProcessor(randomAlphaOfLength(10),
Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)));
Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)), false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(field), equalTo(false));
}
@ -45,8 +46,10 @@ public class RemoveProcessorTests extends ESTestCase {
public void testRemoveNonExistingField() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new RemoveProcessor(randomAlphaOfLength(10),
Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(fieldName)));
Map<String, Object> config = new HashMap<>();
config.put("field", fieldName);
String processorTag = randomAlphaOfLength(10);
Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config);
try {
processor.execute(ingestDocument);
fail("remove field should have failed");
@ -54,4 +57,15 @@ public class RemoveProcessorTests extends ESTestCase {
assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
}
}
public void testIgnoreMissing() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Map<String, Object> config = new HashMap<>();
config.put("field", fieldName);
config.put("ignore_missing", true);
String processorTag = randomAlphaOfLength(10);
Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config);
processor.execute(ingestDocument);
}
}