honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Ingest: Enable Templated Fieldnames in Rename (#31690) 

* Ingest: Enable Templated Fieldnames in Rename
This commit is contained in:
Armin Braun 2018-07-09 13:50:21 +02:00 committed by GitHub
parent 5dcaac32b0
commit 5f5157a2dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 64 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
View File

@ -62,7 +62,7 @@ public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPl
processors.put(DateProcessor.TYPE, new DateProcessor.Factory(parameters.scriptService));
processors.put(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService));
processors.put(AppendProcessor.TYPE, new AppendProcessor.Factory(parameters.scriptService));
processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory());
processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory(parameters.scriptService));
processors.put(RemoveProcessor.TYPE, new RemoveProcessor.Factory(parameters.scriptService));
processors.put(SplitProcessor.TYPE, new SplitProcessor.Factory());
processors.put(JoinProcessor.TYPE, new JoinProcessor.Factory());

43
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
View File

@ -23,6 +23,8 @@ import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import java.util.Map;
@ -33,22 +35,22 @@ public final class RenameProcessor extends AbstractProcessor {
public static final String TYPE = "rename";
private final String field;
private final String targetField;
private final TemplateScript.Factory field;
private final TemplateScript.Factory targetField;
private final boolean ignoreMissing;
RenameProcessor(String tag, String field, String targetField, boolean ignoreMissing) {
RenameProcessor(String tag, TemplateScript.Factory field, TemplateScript.Factory targetField, boolean ignoreMissing) {
super(tag);
this.field = field;
this.targetField = targetField;
this.ignoreMissing = ignoreMissing;
}
String getField() {
TemplateScript.Factory getField() {
return field;
}
String getTargetField() {
TemplateScript.Factory getTargetField() {
return targetField;
}
@ -58,28 +60,30 @@ public final class RenameProcessor extends AbstractProcessor {
@Override
public void execute(IngestDocument document) {
if (document.hasField(field, true) == false) {
String path = document.renderTemplate(field);
if (document.hasField(path, true) == false) {
if (ignoreMissing) {
return;
} else {
throw new IllegalArgumentException("field [" + field + "] doesn't exist");
throw new IllegalArgumentException("field [" + path + "] doesn't exist");
}
}
// We fail here if the target field point to an array slot that is out of range.
// If we didn't do this then we would fail if we set the value in the target_field
// and then on failure processors would not see that value we tried to rename as we already
// removed it.
if (document.hasField(targetField, true)) {
throw new IllegalArgumentException("field [" + targetField + "] already exists");
String target = document.renderTemplate(targetField);
if (document.hasField(target, true)) {
throw new IllegalArgumentException("field [" + target + "] already exists");
}
Object value = document.getFieldValue(field, Object.class);
document.removeField(field);
Object value = document.getFieldValue(path, Object.class);
document.removeField(path);
try {
document.setFieldValue(targetField, value);
document.setFieldValue(target, value);
} catch (Exception e) {
// setting the value back to the original field shouldn't as we just fetched it from that field:
document.setFieldValue(field, value);
document.setFieldValue(path, value);
throw e;
}
}
@ -90,13 +94,24 @@ public final class RenameProcessor extends AbstractProcessor {
}
public static final class Factory implements Processor.Factory {
private final ScriptService scriptService;
public Factory(ScriptService scriptService) {
this.scriptService = scriptService;
}
@Override
public RenameProcessor create(Map<String, Processor.Factory> registry, String processorTag,
Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
"field", field, scriptService);
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
TemplateScript.Factory targetFieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
"target_field", targetField, scriptService);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new RenameProcessor(processorTag, field, targetField, ignoreMissing);
return new RenameProcessor(processorTag, fieldTemplate, targetFieldTemplate , ignoreMissing);
}
}
}

4
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.ingest.common;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import java.util.Collections;
@ -86,7 +87,8 @@ public class DotExpanderProcessorTests extends ESTestCase {
// so because foo is no branch field but a value field the `foo.bar` field can't be expanded
// into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
IngestDocument document = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor("_tag", "foo", "foo.bar", false);
Processor processor = new RenameProcessor("_tag", new TestTemplateService.MockTemplateScript.Factory("foo"),
new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false);
processor.execute(document);
processor = new DotExpanderProcessor("_tag", null, "foo.bar");
processor.execute(document);

22
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
View File

@ -20,8 +20,11 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@ -29,21 +32,26 @@ import static org.hamcrest.CoreMatchers.equalTo;
public class RenameProcessorFactoryTests extends ESTestCase {
private RenameProcessor.Factory factory;
@Before
public void init() {
factory = new RenameProcessor.Factory(TestTemplateService.instance());
}
public void testCreate() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "old_field");
config.put("target_field", "new_field");
String processorTag = randomAlphaOfLength(10);
RenameProcessor renameProcessor = factory.create(null, processorTag, config);
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
assertThat(renameProcessor.isIgnoreMissing(), equalTo(false));
}
public void testCreateWithIgnoreMissing() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "old_field");
config.put("target_field", "new_field");
@ -51,13 +59,12 @@ public class RenameProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
RenameProcessor renameProcessor = factory.create(null, processorTag, config);
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
}
public void testCreateNoFieldPresent() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("target_field", "new_field");
try {
@ -69,7 +76,6 @@ public class RenameProcessorFactoryTests extends ESTestCase {
}
public void testCreateNoToPresent() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "old_field");
try {

29
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.ingest.common;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import java.util.ArrayList;
@ -45,7 +46,7 @@ public class RenameProcessorTests extends ESTestCase {
do {
newFieldName = RandomDocumentPicks.randomFieldName(random());
} while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
Processor processor = createRenameProcessor(fieldName, newFieldName, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
}
@ -63,7 +64,7 @@ public class RenameProcessorTests extends ESTestCase {
document.put("one", one);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "item", false);
Processor processor = createRenameProcessor("list.0", "item", false);
processor.execute(ingestDocument);
Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
assertThat(actualObject, instanceOf(List.class));
@ -76,7 +77,7 @@ public class RenameProcessorTests extends ESTestCase {
assertThat(actualObject, instanceOf(String.class));
assertThat(actualObject, equalTo("item1"));
processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "list.3", false);
processor = createRenameProcessor("list.0", "list.3", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -91,7 +92,7 @@ public class RenameProcessorTests extends ESTestCase {
public void testRenameNonExistingField() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
Processor processor = createRenameProcessor(fieldName,
RandomDocumentPicks.randomFieldName(random()), false);
try {
processor.execute(ingestDocument);
@ -105,7 +106,7 @@ public class RenameProcessorTests extends ESTestCase {
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
Processor processor = createRenameProcessor(fieldName,
RandomDocumentPicks.randomFieldName(random()), true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
@ -114,7 +115,7 @@ public class RenameProcessorTests extends ESTestCase {
public void testRenameNewFieldAlreadyExists() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
Processor processor = new RenameProcessor(randomAlphaOfLength(10), RandomDocumentPicks.randomExistingFieldName(
Processor processor = createRenameProcessor(RandomDocumentPicks.randomExistingFieldName(
random(), ingestDocument), fieldName, false);
try {
processor.execute(ingestDocument);
@ -129,7 +130,7 @@ public class RenameProcessorTests extends ESTestCase {
String fieldName = RandomDocumentPicks.randomFieldName(random());
ingestDocument.setFieldValue(fieldName, null);
String newFieldName = randomValueOtherThanMany(ingestDocument::hasField, () -> RandomDocumentPicks.randomFieldName(random()));
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
Processor processor = createRenameProcessor(fieldName, newFieldName, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(fieldName), equalTo(false));
assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
@ -149,7 +150,7 @@ public class RenameProcessorTests extends ESTestCase {
source.put("list", Collections.singletonList("item"));
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
Processor processor = createRenameProcessor("list", "new_field", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -173,7 +174,7 @@ public class RenameProcessorTests extends ESTestCase {
source.put("list", Collections.singletonList("item"));
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
Processor processor = createRenameProcessor("list", "new_field", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -188,12 +189,12 @@ public class RenameProcessorTests extends ESTestCase {
Map<String, Object> source = new HashMap<>();
source.put("foo", "bar");
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor1 = new RenameProcessor(randomAlphaOfLength(10), "foo", "foo.bar", false);
Processor processor1 = createRenameProcessor("foo", "foo.bar", false);
processor1.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
Processor processor2 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar", "foo.bar.baz", false);
Processor processor2 = createRenameProcessor( "foo.bar", "foo.bar.baz", false);
processor2.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar",
Collections.singletonMap("baz", "bar"))));
@ -201,9 +202,13 @@ public class RenameProcessorTests extends ESTestCase {
assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
// for fun lets try to restore it (which don't allow today)
Processor processor3 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar.baz", "foo", false);
Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false);
Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("field [foo] already exists"));
}
private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing) {
return new RenameProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(field),
new TestTemplateService.MockTemplateScript.Factory(targetField), ignoreMissing);
}
}