honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add ignore_empty_value parameter in set ingest processor (#57030) (#58108)

This commit is contained in:
Dan Hermann 2020-06-15 08:35:08 -05:00 committed by GitHub
parent c3e6aa65dc
commit 8a910443c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 107 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/reference/ingest/processors/set.asciidoc
View File

@ -11,6 +11,7 @@ its value will be replaced with the provided one.
| `field` | yes | - | The field to insert, upsert, or update. Supports <<accessing-template-fields,template snippets>>.
| `value` | yes | - | The value to be set for the field. Supports <<accessing-template-fields,template snippets>>.
| `override` | no | true | If processor will update fields with pre-existing non-null-valued field. When set to `false`, such fields will not be touched.
| `ignore_empty_value` | no | `false` | If `true` and `value` is a <<accessing-template-fields,template snippet>> that evaluates to `null` or the empty string, the processor quietly exits without modifying the document
include::common-options.asciidoc[]
|======

16
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
View File

@ -40,16 +40,18 @@ public final class SetProcessor extends AbstractProcessor {
private final boolean overrideEnabled;
private final TemplateScript.Factory field;
private final ValueSource value;
private final boolean ignoreEmptyValue;
SetProcessor(String tag, TemplateScript.Factory field, ValueSource value) {
this(tag, field, value, true);
this(tag, field, value, true, false);
}
SetProcessor(String tag, TemplateScript.Factory field, ValueSource value, boolean overrideEnabled) {
SetProcessor(String tag, TemplateScript.Factory field, ValueSource value, boolean overrideEnabled, boolean ignoreEmptyValue) {
super(tag);
this.overrideEnabled = overrideEnabled;
this.field = field;
this.value = value;
this.ignoreEmptyValue = ignoreEmptyValue;
}
public boolean isOverrideEnabled() {
@ -64,10 +66,14 @@ public final class SetProcessor extends AbstractProcessor {
return value;
}
public boolean isIgnoreEmptyValue() {
return ignoreEmptyValue;
}
@Override
public IngestDocument execute(IngestDocument document) {
if (overrideEnabled || document.hasField(field) == false || document.getFieldValue(field, Object.class) == null) {
document.setFieldValue(field, value);
document.setFieldValue(field, value, ignoreEmptyValue);
}
return document;
}
@ -93,11 +99,13 @@ public final class SetProcessor extends AbstractProcessor {
boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override", true);
TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
"field", field, scriptService);
boolean ignoreEmptyValue = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_empty_value", false);
return new SetProcessor(
processorTag,
compiledTemplate,
ValueSource.wrap(value, scriptService),
overrideEnabled);
overrideEnabled,
ignoreEmptyValue);
}
}
}

26
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
View File

@ -38,7 +38,7 @@ public class SetProcessorTests extends ESTestCase {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
Processor processor = createSetProcessor(fieldName, fieldValue, true);
Processor processor = createSetProcessor(fieldName, fieldValue, true, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(fieldName), equalTo(true));
assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
@ -50,7 +50,7 @@ public class SetProcessorTests extends ESTestCase {
IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
String fieldName = RandomDocumentPicks.addRandomField(random(), testIngestDocument, fieldValue);
Processor processor = createSetProcessor(fieldName, fieldValue, true);
Processor processor = createSetProcessor(fieldName, fieldValue, true, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(fieldName), equalTo(true));
assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
@ -59,7 +59,7 @@ public class SetProcessorTests extends ESTestCase {
public void testSetFieldsTypeMismatch() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
ingestDocument.setFieldValue("field", "value");
Processor processor = createSetProcessor("field.inner", "value", true);
Processor processor = createSetProcessor("field.inner", "value", true, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -73,7 +73,7 @@ public class SetProcessorTests extends ESTestCase {
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
Processor processor = createSetProcessor(fieldName, fieldValue, false);
Processor processor = createSetProcessor(fieldName, fieldValue, false, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(fieldName), equalTo(true));
assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
@ -83,7 +83,7 @@ public class SetProcessorTests extends ESTestCase {
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
Object fieldValue = "foo";
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = createSetProcessor(fieldName, "bar", false);
Processor processor = createSetProcessor(fieldName, "bar", false, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(fieldName), equalTo(true));
assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
@ -94,7 +94,7 @@ public class SetProcessorTests extends ESTestCase {
Object fieldValue = null;
Object newValue = "bar";
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = createSetProcessor(fieldName, newValue, false);
Processor processor = createSetProcessor(fieldName, newValue, false, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(fieldName), equalTo(true));
assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(newValue));
@ -102,7 +102,7 @@ public class SetProcessorTests extends ESTestCase {
public void testSetMetadataExceptVersion() throws Exception {
Metadata randomMetadata = randomFrom(Metadata.INDEX, Metadata.TYPE, Metadata.ID, Metadata.ROUTING);
Processor processor = createSetProcessor(randomMetadata.getFieldName(), "_value", true);
Processor processor = createSetProcessor(randomMetadata.getFieldName(), "_value", true, false);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(randomMetadata.getFieldName(), String.class), Matchers.equalTo("_value"));
@ -110,7 +110,7 @@ public class SetProcessorTests extends ESTestCase {
public void testSetMetadataVersion() throws Exception {
long version = randomNonNegativeLong();
Processor processor = createSetProcessor(Metadata.VERSION.getFieldName(), version, true);
Processor processor = createSetProcessor(Metadata.VERSION.getFieldName(), version, true, false);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(Metadata.VERSION.getFieldName(), Long.class), Matchers.equalTo(version));
@ -118,7 +118,7 @@ public class SetProcessorTests extends ESTestCase {
public void testSetMetadataVersionType() throws Exception {
String versionType = randomFrom("internal", "external", "external_gte");
Processor processor = createSetProcessor(Metadata.VERSION_TYPE.getFieldName(), versionType, true);
Processor processor = createSetProcessor(Metadata.VERSION_TYPE.getFieldName(), versionType, true, false);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(Metadata.VERSION_TYPE.getFieldName(), String.class), Matchers.equalTo(versionType));
@ -126,7 +126,7 @@ public class SetProcessorTests extends ESTestCase {
public void testSetMetadataIfSeqNo() throws Exception {
long ifSeqNo = randomNonNegativeLong();
Processor processor = createSetProcessor(Metadata.IF_SEQ_NO.getFieldName(), ifSeqNo, true);
Processor processor = createSetProcessor(Metadata.IF_SEQ_NO.getFieldName(), ifSeqNo, true, false);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(Metadata.IF_SEQ_NO.getFieldName(), Long.class), Matchers.equalTo(ifSeqNo));
@ -134,14 +134,14 @@ public class SetProcessorTests extends ESTestCase {
public void testSetMetadataIfPrimaryTerm() throws Exception {
long ifPrimaryTerm = randomNonNegativeLong();
Processor processor = createSetProcessor(Metadata.IF_PRIMARY_TERM.getFieldName(), ifPrimaryTerm, true);
Processor processor = createSetProcessor(Metadata.IF_PRIMARY_TERM.getFieldName(), ifPrimaryTerm, true, false);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(Metadata.IF_PRIMARY_TERM.getFieldName(), Long.class), Matchers.equalTo(ifPrimaryTerm));
}
private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled) {
private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled, boolean ignoreEmptyValue) {
return new SetProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(fieldName),
ValueSource.wrap(fieldValue, TestTemplateService.instance()), overrideEnabled);
ValueSource.wrap(fieldValue, TestTemplateService.instance()), overrideEnabled, ignoreEmptyValue);
}
}

55
modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/270_set_processor.yml Normal file
View File

@ -0,0 +1,55 @@
---
teardown:
- do:
ingest.delete_pipeline:
id: "1"
ignore: 404
---
"Test set processor with template value":
- do:
ingest.put_pipeline:
id: "1"
body: >
{
"processors": [
{
"set" : {
"field" : "foo",
"value" : "{{bar}}",
"ignore_empty_value" : true
}
}
]
}
- match: { acknowledged: true }
- do:
index:
index: test
id: 1
pipeline: "1"
body: {
foo: "hello"
}
- do:
index:
index: test
id: 2
pipeline: "1"
body: {
foo: "hello",
bar: ""
}
- do:
get:
index: test
id: 1
- match: { _source.foo: "hello" }
- do:
get:
index: test
id: 2
- match: { _source.foo: "hello" }

26
server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
View File

@ -427,6 +427,32 @@ public final class IngestDocument {
setFieldValue(fieldPathTemplate.newInstance(model).execute(), valueSource.copyAndResolve(model), false);
}
/**
* Sets the provided value to the provided path in the document.
* Any non existing path element will be created. If the last element is a list,
* the value will replace the existing list.
* @param fieldPathTemplate Resolves to the path with dot-notation within the document
* @param valueSource The value source that will produce the value to put in for the path key
* @param ignoreEmptyValue The flag to determine whether to exit quietly when the value produced by TemplatedValue is null or empty
* @throws IllegalArgumentException if the path is null, empty, invalid or if the value cannot be set to the
* item identified by the provided path.
*/
public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource valueSource, boolean ignoreEmptyValue) {
Map<String, Object> model = createTemplateModel();
Object value = valueSource.copyAndResolve(model);
if (ignoreEmptyValue && valueSource instanceof ValueSource.TemplatedValue) {
if (value == null) {
return;
}
String valueStr = (String) value;
if (valueStr.isEmpty()) {
return;
}
}
setFieldValue(fieldPathTemplate.newInstance(model).execute(), value, false);
}
private void setFieldValue(String path, Object value, boolean append) {
FieldPath fieldPath = new FieldPath(path);
Object context = fieldPath.initialContext;