From 2064fe3985a7dd01b5ec73d9f55073ab788d1c27 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 22 Mar 2016 16:46:54 -0700 Subject: [PATCH] add type conversion support to ConvertProcessor --- .../ingest/processor/ConvertProcessor.java | 30 ++++- .../ConvertProcessorFactoryTests.java | 18 ++- .../processor/ConvertProcessorTests.java | 103 +++++++++++++++--- docs/reference/ingest/ingest-node.asciidoc | 15 ++- 4 files changed, 145 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java index b6274ff83e5..1a6ce94e3db 100644 --- a/core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java +++ b/core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java @@ -73,6 +73,23 @@ public final class ConvertProcessor extends AbstractProcessor { public Object convert(Object value) { return value.toString(); } + }, AUTO { + @Override + public Object convert(Object value) { + if (!(value instanceof String)) { + return value; + } + try { + return BOOLEAN.convert(value); + } catch (IllegalArgumentException e) { } + try { + return INTEGER.convert(value); + } catch (IllegalArgumentException e) {} + try { + return FLOAT.convert(value); + } catch (IllegalArgumentException e) {} + return value; + } }; @Override @@ -94,11 +111,13 @@ public final class ConvertProcessor extends AbstractProcessor { public static final String TYPE = "convert"; private final String field; + private final String targetField; private final Type convertType; - ConvertProcessor(String tag, String field, Type convertType) { + ConvertProcessor(String tag, String field, String targetField, Type convertType) { super(tag); this.field = field; + this.targetField = targetField; this.convertType = convertType; } @@ -106,6 +125,10 @@ public final class ConvertProcessor extends AbstractProcessor { return field; } + String getTargetField() { + return targetField; + } + Type getConvertType() { return convertType; } @@ -128,7 +151,7 @@ public final class ConvertProcessor extends AbstractProcessor { } else { newValue = convertType.convert(oldValue); } - document.setFieldValue(field, newValue); + document.setFieldValue(targetField, newValue); } @Override @@ -141,8 +164,9 @@ public final class ConvertProcessor extends AbstractProcessor { public ConvertProcessor doCreate(String processorTag, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type"); + String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field); Type convertType = Type.fromString(processorTag, "type", typeProperty); - return new ConvertProcessor(processorTag, field, convertType); + return new ConvertProcessor(processorTag, field, targetField, convertType); } } } diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java index 831e87436ba..f54f04c0cf8 100644 --- a/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.ingest.processor; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ingest.core.AbstractProcessorFactory; -import org.elasticsearch.ingest.core.Processor; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; @@ -44,6 +43,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase { ConvertProcessor convertProcessor = factory.create(config); assertThat(convertProcessor.getTag(), equalTo(processorTag)); assertThat(convertProcessor.getField(), equalTo("field1")); + assertThat(convertProcessor.getTargetField(), equalTo("field1")); assertThat(convertProcessor.getConvertType(), equalTo(type)); } @@ -88,4 +88,20 @@ public class ConvertProcessorFactoryTests extends ESTestCase { assertThat(e.getMessage(), Matchers.equalTo("[type] required property is missing")); } } + + public void testCreateWithExplicitTargetField() throws Exception { + ConvertProcessor.Factory factory = new ConvertProcessor.Factory(); + Map config = new HashMap<>(); + ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values()); + config.put("field", "field1"); + config.put("target_field", "field2"); + config.put("type", type.toString()); + String processorTag = randomAsciiOfLength(10); + config.put(AbstractProcessorFactory.TAG_KEY, processorTag); + ConvertProcessor convertProcessor = factory.create(config); + assertThat(convertProcessor.getTag(), equalTo(processorTag)); + assertThat(convertProcessor.getField(), equalTo("field1")); + assertThat(convertProcessor.getTargetField(), equalTo("field2")); + assertThat(convertProcessor.getConvertType(), equalTo(type)); + } } diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java index 1350ebab601..936875cf73a 100644 --- a/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java @@ -34,6 +34,7 @@ import java.util.Map; import static org.elasticsearch.ingest.processor.ConvertProcessor.Type; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; public class ConvertProcessorTests extends ESTestCase { @@ -41,7 +42,7 @@ public class ConvertProcessorTests extends ESTestCase { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); int randomInt = randomInt(); String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt)); } @@ -57,7 +58,7 @@ public class ConvertProcessorTests extends ESTestCase { expectedList.add(randomInt); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -68,7 +69,7 @@ public class ConvertProcessorTests extends ESTestCase { String value = "string-" + randomAsciiOfLengthBetween(1, 10); ingestDocument.setFieldValue(fieldName, value); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -84,7 +85,7 @@ public class ConvertProcessorTests extends ESTestCase { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat); expectedResult.put(fieldName, randomFloat); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat)); } @@ -100,7 +101,7 @@ public class ConvertProcessorTests extends ESTestCase { expectedList.add(randomFloat); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -111,7 +112,7 @@ public class ConvertProcessorTests extends ESTestCase { String value = "string-" + randomAsciiOfLengthBetween(1, 10); ingestDocument.setFieldValue(fieldName, value); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -129,7 +130,7 @@ public class ConvertProcessorTests extends ESTestCase { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean)); } @@ -149,7 +150,7 @@ public class ConvertProcessorTests extends ESTestCase { expectedList.add(randomBoolean); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -166,7 +167,7 @@ public class ConvertProcessorTests extends ESTestCase { } ingestDocument.setFieldValue(fieldName, fieldValue); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -200,7 +201,7 @@ public class ConvertProcessorTests extends ESTestCase { } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.STRING); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue)); } @@ -236,7 +237,7 @@ public class ConvertProcessorTests extends ESTestCase { expectedList.add(randomValueString); } String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.STRING); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); } @@ -245,7 +246,7 @@ public class ConvertProcessorTests extends ESTestCase { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); Type type = randomFrom(Type.values()); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, type); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -257,7 +258,7 @@ public class ConvertProcessorTests extends ESTestCase { public void testConvertNullField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null)); Type type = randomFrom(Type.values()); - Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", type); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -265,4 +266,80 @@ public class ConvertProcessorTests extends ESTestCase { assertThat(e.getMessage(), equalTo("Field [field] is null, cannot be converted to type [" + type + "]")); } } + + public void testAutoConvertNotString() throws Exception { + Object randomValue; + switch(randomIntBetween(0, 2)) { + case 0: + float randomFloat = randomFloat(); + randomValue = randomFloat; + break; + case 1: + int randomInt = randomInt(); + randomValue = randomInt; + break; + case 2: + boolean randomBoolean = randomBoolean(); + randomValue = randomBoolean; + break; + default: + throw new UnsupportedOperationException(); + } + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue)); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO); + processor.execute(ingestDocument); + Object convertedValue = ingestDocument.getFieldValue("field", Object.class); + assertThat(convertedValue, sameInstance(randomValue)); + } + + public void testAutoConvertStringNotMatched() throws Exception { + String value = "notAnIntFloatOrBool"; + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value)); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO); + processor.execute(ingestDocument); + Object convertedValue = ingestDocument.getFieldValue("field", Object.class); + assertThat(convertedValue, sameInstance(value)); + } + + public void testAutoConvertMatchBoolean() throws Exception { + boolean randomBoolean = randomBoolean(); + String booleanString = Boolean.toString(randomBoolean); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", booleanString)); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO); + processor.execute(ingestDocument); + Object convertedValue = ingestDocument.getFieldValue("field", Object.class); + assertThat(convertedValue, equalTo(randomBoolean)); + } + + public void testAutoConvertMatchInteger() throws Exception { + int randomInt = randomInt(); + String randomString = Integer.toString(randomInt); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString)); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO); + processor.execute(ingestDocument); + Object convertedValue = ingestDocument.getFieldValue("field", Object.class); + assertThat(convertedValue, equalTo(randomInt)); + } + + public void testAutoConvertMatchFloat() throws Exception { + float randomFloat = randomFloat(); + String randomString = Float.toString(randomFloat); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString)); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO); + processor.execute(ingestDocument); + Object convertedValue = ingestDocument.getFieldValue("field", Object.class); + assertThat(convertedValue, equalTo(randomFloat)); + } + + public void testTargetField() throws Exception { + IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>()); + int randomInt = randomInt(); + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, String.valueOf(randomInt)); + String targetField = fieldName + randomAsciiOfLength(5); + Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt))); + assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt)); + + } } diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 10b640dbaf1..146e033736a 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -668,18 +668,25 @@ Accepts a single value or an array of values. Converts an existing field's value to a different type, such as converting a string to an integer. If the field value is an array, all members will be converted. -The supported types include: `integer`, `float`, `string`, and `boolean`. +The supported types include: `integer`, `float`, `string`, `boolean`, and `auto`. Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise. +Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type. +For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And +a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot +be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In +such a case, `target_field` will still be updated with the unconverted field value. + [[convert-options]] .Convert Options [options="header"] |====== -| Name | Required | Default | Description -| `field` | yes | - | The field whose value is to be converted -| `type` | yes | - | The type to convert the existing value to +| Name | Required | Default | Description +| `field` | yes | - | The field whose value is to be converted +| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place +| `type` | yes | - | The type to convert the existing value to |====== [source,js]