honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable convert processor to support Long and Double. (#27957) 

Closes #23085
This commit is contained in:
Sian Lerk Lau 2018-01-03 18:27:55 +08:00 committed by Martijn van Groningen
parent 6f52fbeac6
commit 5e3ba8a88d
3 changed files with 150 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/reference/ingest/ingest-node.asciidoc
View File

@ -778,16 +778,17 @@ Accepts a single value or an array of values.
Converts an existing field's value to a different type, such as converting a string to an integer. Converts an existing field's value to a different type, such as converting a string to an integer.
If the field value is an array, all members will be converted. If the field value is an array, all members will be converted.
The supported types include: `integer`, `float`, `string`, `boolean`, and `auto`. The supported types include: `integer`, `long`, `float`, `double`, `string`, `boolean`, and `auto`.
Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to
false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise. false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise.
Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type. Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type.
For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. Do note
a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot that float takes precedence of double in `auto`. A value of `"242.15"` will "automatically" be converted to
be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In `242.15` of type `float`. If a provided field cannot be appropriately converted, the Convert Processor will
such a case, `target_field` will still be updated with the unconverted field value. still process successfully and leave the field value as-is. In such a case, `target_field` will
still be updated with the unconverted field value.
[[convert-options]] [[convert-options]]
.Convert Options .Convert Options

26
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
View File

@ -48,6 +48,24 @@ public final class ConvertProcessor extends AbstractProcessor {
} }
} }
}, LONG {
@Override
public Object convert(Object value) {
try {
return Long.parseLong(value.toString());
} catch(NumberFormatException e) {
throw new IllegalArgumentException("unable to convert [" + value + "] to long", e);
}
}
}, DOUBLE {
@Override
public Object convert(Object value) {
try {
return Double.parseDouble(value.toString());
} catch(NumberFormatException e) {
throw new IllegalArgumentException("unable to convert [" + value + "] to double", e);
}
}
}, FLOAT { }, FLOAT {
@Override @Override
public Object convert(Object value) { public Object convert(Object value) {
@ -81,13 +99,19 @@ public final class ConvertProcessor extends AbstractProcessor {
} }
try { try {
return BOOLEAN.convert(value); return BOOLEAN.convert(value);
} catch (IllegalArgumentException e) { } } catch (IllegalArgumentException e) {}
try { try {
return INTEGER.convert(value); return INTEGER.convert(value);
} catch (IllegalArgumentException e) {} } catch (IllegalArgumentException e) {}
try {
return LONG.convert(value);
} catch (IllegalArgumentException e) {}
try { try {
return FLOAT.convert(value); return FLOAT.convert(value);
} catch (IllegalArgumentException e) {} } catch (IllegalArgumentException e) {}
try {
return DOUBLE.convert(value);
} catch (IllegalArgumentException e) {}
return value; return value;
} }
}; };

119
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
View File

@ -36,6 +36,7 @@ import static org.elasticsearch.ingest.common.ConvertProcessor.Type;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.Matchers.sameInstance;
import static org.hamcrest.Matchers.not;
public class ConvertProcessorTests extends ESTestCase { public class ConvertProcessorTests extends ESTestCase {
@ -79,6 +80,92 @@ public class ConvertProcessorTests extends ESTestCase {
} }
} }
public void testConvertLong() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
Map<String, Long> expectedResult = new HashMap<>();
long randomLong = randomLong();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomLong);
expectedResult.put(fieldName, randomLong);
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(randomLong));
}
public void testConvertLongList() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
int numItems = randomIntBetween(1, 10);
List<String> fieldValue = new ArrayList<>();
List<Long> expectedList = new ArrayList<>();
for (int j = 0; j < numItems; j++) {
long randomLong = randomLong();
fieldValue.add(Long.toString(randomLong));
expectedList.add(randomLong);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
public void testConvertLongError() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
String value = "string-" + randomAlphaOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.LONG, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
} catch(IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to long"));
}
}
public void testConvertDouble() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
Map<String, Double> expectedResult = new HashMap<>();
double randomDouble = randomDouble();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomDouble);
expectedResult.put(fieldName, randomDouble);
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Double.class), equalTo(randomDouble));
}
public void testConvertDoubleList() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
int numItems = randomIntBetween(1, 10);
List<String> fieldValue = new ArrayList<>();
List<Double> expectedList = new ArrayList<>();
for (int j = 0; j < numItems; j++) {
double randomDouble = randomDouble();
fieldValue.add(Double.toString(randomDouble));
expectedList.add(randomDouble);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
public void testConvertDoubleError() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
String value = "string-" + randomAlphaOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), fieldName, fieldName, Type.DOUBLE, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
} catch(IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("unable to convert [" + value + "] to double"));
}
}
public void testConvertFloat() throws Exception { public void testConvertFloat() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
Map<String, Float> expectedResult = new HashMap<>(); Map<String, Float> expectedResult = new HashMap<>();
@ -231,6 +318,16 @@ public class ConvertProcessorTests extends ESTestCase {
randomValue = randomBoolean; randomValue = randomBoolean;
randomValueString = Boolean.toString(randomBoolean); randomValueString = Boolean.toString(randomBoolean);
break; break;
case 3:
long randomLong = randomLong();
randomValue = randomLong;
randomValueString = Long.toString(randomLong);
break;
case 4:
double randomDouble = randomDouble();
randomValue = randomDouble;
randomValueString = Double.toString(randomDouble);
break;
default: default:
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@ -342,6 +439,28 @@ public class ConvertProcessorTests extends ESTestCase {
assertThat(convertedValue, equalTo(randomInt)); assertThat(convertedValue, equalTo(randomInt));
} }
public void testAutoConvertMatchLong() throws Exception {
long randomLong = randomLong();
String randomString = Long.toString(randomLong);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomLong));
}
public void testAutoConvertDoubleNotMatched() throws Exception {
double randomDouble = randomDouble();
String randomString = Double.toString(randomDouble);
float randomFloat = Float.parseFloat(randomString);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, not(randomDouble));
assertThat(convertedValue, equalTo(randomFloat));
}
public void testAutoConvertMatchFloat() throws Exception { public void testAutoConvertMatchFloat() throws Exception {
float randomFloat = randomFloat(); float randomFloat = randomFloat();
String randomString = Float.toString(randomFloat); String randomString = Float.toString(randomFloat);