honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add type conversion support to ConvertProcessor

This commit is contained in:
Tal Levy 2016-03-22 16:46:54 -07:00
parent 3e9f8a4c59
commit 2064fe3985
4 changed files with 145 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java
View File

@ -73,6 +73,23 @@ public final class ConvertProcessor extends AbstractProcessor {
public Object convert(Object value) {
return value.toString();
}
}, AUTO {
@Override
public Object convert(Object value) {
if (!(value instanceof String)) {
return value;
}
try {
return BOOLEAN.convert(value);
} catch (IllegalArgumentException e) { }
try {
return INTEGER.convert(value);
} catch (IllegalArgumentException e) {}
try {
return FLOAT.convert(value);
} catch (IllegalArgumentException e) {}
return value;
}
};
@Override
@ -94,11 +111,13 @@ public final class ConvertProcessor extends AbstractProcessor {
public static final String TYPE = "convert";
private final String field;
private final String targetField;
private final Type convertType;
ConvertProcessor(String tag, String field, Type convertType) {
ConvertProcessor(String tag, String field, String targetField, Type convertType) {
super(tag);
this.field = field;
this.targetField = targetField;
this.convertType = convertType;
}
@ -106,6 +125,10 @@ public final class ConvertProcessor extends AbstractProcessor {
return field;
}
String getTargetField() {
return targetField;
}
Type getConvertType() {
return convertType;
}
@ -128,7 +151,7 @@ public final class ConvertProcessor extends AbstractProcessor {
} else {
newValue = convertType.convert(oldValue);
}
document.setFieldValue(field, newValue);
document.setFieldValue(targetField, newValue);
}
@Override
@ -141,8 +164,9 @@ public final class ConvertProcessor extends AbstractProcessor {
public ConvertProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
Type convertType = Type.fromString(processorTag, "type", typeProperty);
return new ConvertProcessor(processorTag, field, convertType);
return new ConvertProcessor(processorTag, field, targetField, convertType);
}
}
}

18
core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.ingest.processor;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ingest.core.AbstractProcessorFactory;
import org.elasticsearch.ingest.core.Processor;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
@ -44,6 +43,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
ConvertProcessor convertProcessor = factory.create(config);
assertThat(convertProcessor.getTag(), equalTo(processorTag));
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field1"));
assertThat(convertProcessor.getConvertType(), equalTo(type));
}
@ -88,4 +88,20 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
assertThat(e.getMessage(), Matchers.equalTo("[type] required property is missing"));
}
}
public void testCreateWithExplicitTargetField() throws Exception {
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
Map<String, Object> config = new HashMap<>();
ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
config.put("field", "field1");
config.put("target_field", "field2");
config.put("type", type.toString());
String processorTag = randomAsciiOfLength(10);
config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
ConvertProcessor convertProcessor = factory.create(config);
assertThat(convertProcessor.getTag(), equalTo(processorTag));
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field2"));
assertThat(convertProcessor.getConvertType(), equalTo(type));
}
}

103
core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java
View File

@ -34,6 +34,7 @@ import java.util.Map;
import static org.elasticsearch.ingest.processor.ConvertProcessor.Type;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.sameInstance;
public class ConvertProcessorTests extends ESTestCase {
@ -41,7 +42,7 @@ public class ConvertProcessorTests extends ESTestCase {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
int randomInt = randomInt();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
}
@ -57,7 +58,7 @@ public class ConvertProcessorTests extends ESTestCase {
expectedList.add(randomInt);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
@ -68,7 +69,7 @@ public class ConvertProcessorTests extends ESTestCase {
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -84,7 +85,7 @@ public class ConvertProcessorTests extends ESTestCase {
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat);
expectedResult.put(fieldName, randomFloat);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat));
}
@ -100,7 +101,7 @@ public class ConvertProcessorTests extends ESTestCase {
expectedList.add(randomFloat);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
@ -111,7 +112,7 @@ public class ConvertProcessorTests extends ESTestCase {
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -129,7 +130,7 @@ public class ConvertProcessorTests extends ESTestCase {
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean));
}
@ -149,7 +150,7 @@ public class ConvertProcessorTests extends ESTestCase {
expectedList.add(randomBoolean);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
@ -166,7 +167,7 @@ public class ConvertProcessorTests extends ESTestCase {
}
ingestDocument.setFieldValue(fieldName, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -200,7 +201,7 @@ public class ConvertProcessorTests extends ESTestCase {
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.STRING);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue));
}
@ -236,7 +237,7 @@ public class ConvertProcessorTests extends ESTestCase {
expectedList.add(randomValueString);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.STRING);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
@ -245,7 +246,7 @@ public class ConvertProcessorTests extends ESTestCase {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Type type = randomFrom(Type.values());
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, type);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -257,7 +258,7 @@ public class ConvertProcessorTests extends ESTestCase {
public void testConvertNullField() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
Type type = randomFrom(Type.values());
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", type);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -265,4 +266,80 @@ public class ConvertProcessorTests extends ESTestCase {
assertThat(e.getMessage(), equalTo("Field [field] is null, cannot be converted to type [" + type + "]"));
}
}
public void testAutoConvertNotString() throws Exception {
Object randomValue;
switch(randomIntBetween(0, 2)) {
case 0:
float randomFloat = randomFloat();
randomValue = randomFloat;
break;
case 1:
int randomInt = randomInt();
randomValue = randomInt;
break;
case 2:
boolean randomBoolean = randomBoolean();
randomValue = randomBoolean;
break;
default:
throw new UnsupportedOperationException();
}
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, sameInstance(randomValue));
}
public void testAutoConvertStringNotMatched() throws Exception {
String value = "notAnIntFloatOrBool";
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, sameInstance(value));
}
public void testAutoConvertMatchBoolean() throws Exception {
boolean randomBoolean = randomBoolean();
String booleanString = Boolean.toString(randomBoolean);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", booleanString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomBoolean));
}
public void testAutoConvertMatchInteger() throws Exception {
int randomInt = randomInt();
String randomString = Integer.toString(randomInt);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomInt));
}
public void testAutoConvertMatchFloat() throws Exception {
float randomFloat = randomFloat();
String randomString = Float.toString(randomFloat);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomFloat));
}
public void testTargetField() throws Exception {
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
int randomInt = randomInt();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, String.valueOf(randomInt));
String targetField = fieldName + randomAsciiOfLength(5);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt)));
assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt));
}
}

15
docs/reference/ingest/ingest-node.asciidoc
View File

@ -668,18 +668,25 @@ Accepts a single value or an array of values.
Converts an existing field's value to a different type, such as converting a string to an integer.
If the field value is an array, all members will be converted.
The supported types include: `integer`, `float`, `string`, and `boolean`.
The supported types include: `integer`, `float`, `string`, `boolean`, and `auto`.
Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to
false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise.
Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type.
For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And
a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot
be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In
such a case, `target_field` will still be updated with the unconverted field value.
[[convert-options]]
.Convert Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field whose value is to be converted
| `type` | yes | - | The type to convert the existing value to
| Name | Required | Default | Description
| `field` | yes | - | The field whose value is to be converted
| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place
| `type` | yes | - | The type to convert the existing value to
|======
[source,js]