update all processors to only operate on one field at a time when possible
This commit is contained in:
parent
d7c3b51b9c
commit
45f48ac126
|
@ -4,52 +4,46 @@
|
||||||
=== Processors
|
=== Processors
|
||||||
|
|
||||||
==== Set processor
|
==== Set processor
|
||||||
Sets one or more fields and associates them with the specified values. If a field already exists,
|
Sets one field and associates it with the specified value. If a field already exists,
|
||||||
its value will be replaced with the provided one.
|
its value will be replaced with the provided one.
|
||||||
|
|
||||||
[source,js]
|
[source,js]
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"set": {
|
"set": {
|
||||||
"fields": {
|
"field1": 582.1
|
||||||
"field": 582.1
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
|
||||||
==== Remove processor
|
==== Remove processor
|
||||||
Removes one or more existing fields. If a field doesn't exist, nothing will happen.
|
Removes an existing field. If a field doesn't exist, nothing will happen.
|
||||||
|
|
||||||
[source,js]
|
[source,js]
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"remove": {
|
"remove": {
|
||||||
"fields": [
|
"field": "foo"
|
||||||
"field1","field2"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
|
||||||
==== Rename processor
|
==== Rename processor
|
||||||
Renames one or more existing fields. If a field doesn't exist, an exception will be thrown. Also, the new field
|
Renames an existing fields. If a field doesn't exist, an exception will be thrown. Also, the new field
|
||||||
name must not exist.
|
name must not exist.
|
||||||
|
|
||||||
[source,js]
|
[source,js]
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"rename": {
|
"rename": {
|
||||||
"fields": {
|
"field": "foo"
|
||||||
"field1": "field2"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
==== Convert processor
|
==== Convert processor
|
||||||
Converts one or more field value to a different type, like turning a string to an integer.
|
Converts an existing field's value to a different type, like turning a string to an integer.
|
||||||
If the field value is an array, all members will be converted.
|
If the field value is an array, all members will be converted.
|
||||||
|
|
||||||
The supported types include: `integer`, `float`, `string`, and `boolean`.
|
The supported types include: `integer`, `float`, `string`, and `boolean`.
|
||||||
|
@ -61,10 +55,7 @@ false if its string value is equal to `false` (ignore case) and it will throw ex
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"convert": {
|
"convert": {
|
||||||
"fields": {
|
"foo": "integer"
|
||||||
"field1": "integer",
|
|
||||||
"field2": "float"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
@ -73,8 +64,7 @@ false if its string value is equal to `false` (ignore case) and it will throw ex
|
||||||
Converts a string field by applying a regular expression and a replacement.
|
Converts a string field by applying a regular expression and a replacement.
|
||||||
If the field is not a string, the processor will throw an exception.
|
If the field is not a string, the processor will throw an exception.
|
||||||
|
|
||||||
This configuration takes an `expression` array consisting of objects. Each object
|
This configuration takes a `field` for the field name, `pattern` for the
|
||||||
holds three elements: `field` for the field name, `pattern` for the
|
|
||||||
pattern to be replaced, and `replacement` for the string to replace the matching patterns with.
|
pattern to be replaced, and `replacement` for the string to replace the matching patterns with.
|
||||||
|
|
||||||
|
|
||||||
|
@ -82,13 +72,9 @@ pattern to be replaced, and `replacement` for the string to replace the matching
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"gsub": {
|
"gsub": {
|
||||||
"expressions": [
|
"field": "field1",
|
||||||
{
|
"pattern": "\.",
|
||||||
"field": "field1",
|
"replacement": "-"
|
||||||
"pattern": "\.",
|
|
||||||
"replacement": "-"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
@ -101,9 +87,8 @@ Throws error when the field is not an array.
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"join": {
|
"join": {
|
||||||
"fields": {
|
"field": "joined_array_field",
|
||||||
"joined_array_field": "other_array_field"
|
"separator": "-"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
@ -115,9 +100,7 @@ Split a field to an array using a separator character. Only works on string fiel
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"split": {
|
"split": {
|
||||||
"fields": {
|
"field": ","
|
||||||
"message": ","
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
@ -129,7 +112,7 @@ Converts a string to its lowercase equivalent.
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"lowercase": {
|
"lowercase": {
|
||||||
"fields": ["foo", "bar"]
|
"field": "foo"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
@ -141,7 +124,7 @@ Converts a string to its uppercase equivalent.
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"uppercase": {
|
"uppercase": {
|
||||||
"fields": ["foo", "bar"]
|
"field": "foo"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
@ -153,7 +136,7 @@ Trims whitespace from field. NOTE: this only works on leading and trailing white
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
{
|
{
|
||||||
"trim": {
|
"trim": {
|
||||||
"fields": ["foo", "bar"]
|
"field": "foo"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
|
@ -538,4 +521,4 @@ The delete pipeline api deletes pipelines by id.
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
DELETE _ingest/pipeline/my-pipeline-id
|
DELETE _ingest/pipeline/my-pipeline-id
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
// AUTOSENSE
|
// AUTOSENSE
|
||||||
|
|
|
@ -32,25 +32,23 @@ import java.util.Map;
|
||||||
*/
|
*/
|
||||||
public abstract class AbstractStringProcessor implements Processor {
|
public abstract class AbstractStringProcessor implements Processor {
|
||||||
|
|
||||||
private final Collection<String> fields;
|
private final String field;
|
||||||
|
|
||||||
protected AbstractStringProcessor(Collection<String> fields) {
|
protected AbstractStringProcessor(String field) {
|
||||||
this.fields = fields;
|
this.field = field;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Collection<String> getFields() {
|
public String getField() {
|
||||||
return fields;
|
return field;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public final void execute(IngestDocument document) {
|
public final void execute(IngestDocument document) {
|
||||||
for(String field : fields) {
|
String val = document.getFieldValue(field, String.class);
|
||||||
String val = document.getFieldValue(field, String.class);
|
if (val == null) {
|
||||||
if (val == null) {
|
throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
|
||||||
throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
|
|
||||||
}
|
|
||||||
document.setFieldValue(field, process(val));
|
|
||||||
}
|
}
|
||||||
|
document.setFieldValue(field, process(val));
|
||||||
}
|
}
|
||||||
|
|
||||||
protected abstract String process(String value);
|
protected abstract String process(String value);
|
||||||
|
@ -58,10 +56,10 @@ public abstract class AbstractStringProcessor implements Processor {
|
||||||
public static abstract class Factory<T extends AbstractStringProcessor> implements Processor.Factory<T> {
|
public static abstract class Factory<T extends AbstractStringProcessor> implements Processor.Factory<T> {
|
||||||
@Override
|
@Override
|
||||||
public T create(Map<String, Object> config) throws Exception {
|
public T create(Map<String, Object> config) throws Exception {
|
||||||
List<String> fields = ConfigurationUtils.readList(config, "fields");
|
String field = ConfigurationUtils.readStringProperty(config, "field");
|
||||||
return newProcessor(Collections.unmodifiableList(fields));
|
return newProcessor(field);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected abstract T newProcessor(Collection<String> fields);
|
protected abstract T newProcessor(String field);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -149,4 +149,15 @@ public final class ConfigurationUtils {
|
||||||
throw new IllegalArgumentException("property [" + propertyName + "] isn't a map, but of type [" + value.getClass().getName() + "]");
|
throw new IllegalArgumentException("property [" + propertyName + "] isn't a map, but of type [" + value.getClass().getName() + "]");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns and removes the specified property as an {@link Object} from the specified configuration map.
|
||||||
|
*/
|
||||||
|
public static Object readObject(Map<String, Object> configuration, String propertyName) {
|
||||||
|
Object value = configuration.remove(propertyName);
|
||||||
|
if (value == null) {
|
||||||
|
throw new IllegalArgumentException("required property [" + propertyName + "] is missing");
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -87,38 +87,41 @@ public class ConvertProcessor implements Processor {
|
||||||
|
|
||||||
public static final String TYPE = "convert";
|
public static final String TYPE = "convert";
|
||||||
|
|
||||||
private final Map<String, Type> fields;
|
private final String field;
|
||||||
|
private final Type convertType;
|
||||||
|
|
||||||
ConvertProcessor(Map<String, Type> fields) {
|
ConvertProcessor(String field, Type convertType) {
|
||||||
this.fields = fields;
|
this.field = field;
|
||||||
|
this.convertType = convertType;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, Type> getFields() {
|
String getField() {
|
||||||
return fields;
|
return field;
|
||||||
|
}
|
||||||
|
|
||||||
|
Type getConvertType() {
|
||||||
|
return convertType;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void execute(IngestDocument document) {
|
public void execute(IngestDocument document) {
|
||||||
for(Map.Entry<String, Type> entry : fields.entrySet()) {
|
Object oldValue = document.getFieldValue(field, Object.class);
|
||||||
Type type = entry.getValue();
|
Object newValue;
|
||||||
Object oldValue = document.getFieldValue(entry.getKey(), Object.class);
|
if (oldValue == null) {
|
||||||
Object newValue;
|
throw new IllegalArgumentException("Field [" + field + "] is null, cannot be converted to type [" + convertType + "]");
|
||||||
if (oldValue == null) {
|
|
||||||
throw new IllegalArgumentException("Field [" + entry.getKey() + "] is null, cannot be converted to type [" + type + "]");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (oldValue instanceof List) {
|
|
||||||
List<?> list = (List<?>) oldValue;
|
|
||||||
List<Object> newList = new ArrayList<>();
|
|
||||||
for (Object value : list) {
|
|
||||||
newList.add(type.convert(value));
|
|
||||||
}
|
|
||||||
newValue = newList;
|
|
||||||
} else {
|
|
||||||
newValue = type.convert(oldValue);
|
|
||||||
}
|
|
||||||
document.setFieldValue(entry.getKey(), newValue);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (oldValue instanceof List) {
|
||||||
|
List<?> list = (List<?>) oldValue;
|
||||||
|
List<Object> newList = new ArrayList<>();
|
||||||
|
for (Object value : list) {
|
||||||
|
newList.add(convertType.convert(value));
|
||||||
|
}
|
||||||
|
newValue = newList;
|
||||||
|
} else {
|
||||||
|
newValue = convertType.convert(oldValue);
|
||||||
|
}
|
||||||
|
document.setFieldValue(field, newValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -129,12 +132,9 @@ public class ConvertProcessor implements Processor {
|
||||||
public static class Factory implements Processor.Factory<ConvertProcessor> {
|
public static class Factory implements Processor.Factory<ConvertProcessor> {
|
||||||
@Override
|
@Override
|
||||||
public ConvertProcessor create(Map<String, Object> config) throws Exception {
|
public ConvertProcessor create(Map<String, Object> config) throws Exception {
|
||||||
Map<String, String> fields = ConfigurationUtils.readMap(config, "fields");
|
String field = ConfigurationUtils.readStringProperty(config, "field");
|
||||||
Map<String, Type> convertFields = new HashMap<>();
|
Type convertType = Type.fromString(ConfigurationUtils.readStringProperty(config, "type"));
|
||||||
for (Map.Entry<String, String> entry : fields.entrySet()) {
|
return new ConvertProcessor(field, convertType);
|
||||||
convertFields.put(entry.getKey(), Type.fromString(entry.getValue()));
|
|
||||||
}
|
|
||||||
return new ConvertProcessor(Collections.unmodifiableMap(convertFields));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,70 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to Elasticsearch under one or more contributor
|
|
||||||
* license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright
|
|
||||||
* ownership. Elasticsearch licenses this file to you under
|
|
||||||
* the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
* not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing,
|
|
||||||
* software distributed under the License is distributed on an
|
|
||||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
* KIND, either express or implied. See the License for the
|
|
||||||
* specific language governing permissions and limitations
|
|
||||||
* under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.elasticsearch.ingest.processor.gsub;
|
|
||||||
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Represents a gsub expression containing the field name, the pattern to look for and its string replacement.
|
|
||||||
*/
|
|
||||||
public class GsubExpression {
|
|
||||||
|
|
||||||
private final String fieldName;
|
|
||||||
private final Pattern pattern;
|
|
||||||
private final String replacement;
|
|
||||||
|
|
||||||
public GsubExpression(String fieldName, Pattern pattern, String replacement) {
|
|
||||||
this.fieldName = Objects.requireNonNull(fieldName);
|
|
||||||
this.pattern = Objects.requireNonNull(pattern);
|
|
||||||
this.replacement = Objects.requireNonNull(replacement);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getFieldName() {
|
|
||||||
return fieldName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Pattern getPattern() {
|
|
||||||
return pattern;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getReplacement() {
|
|
||||||
return replacement;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean equals(Object o) {
|
|
||||||
if (this == o) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (o == null || getClass() != o.getClass()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
GsubExpression that = (GsubExpression) o;
|
|
||||||
return Objects.equals(fieldName, that.fieldName) &&
|
|
||||||
Objects.equals(pattern.pattern(), that.pattern.pattern()) &&
|
|
||||||
Objects.equals(replacement, that.replacement);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int hashCode() {
|
|
||||||
return Objects.hash(fieldName, pattern, replacement);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -23,8 +23,6 @@ import org.elasticsearch.ingest.IngestDocument;
|
||||||
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
||||||
import org.elasticsearch.ingest.processor.Processor;
|
import org.elasticsearch.ingest.processor.Processor;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
@ -37,27 +35,38 @@ public class GsubProcessor implements Processor {
|
||||||
|
|
||||||
public static final String TYPE = "gsub";
|
public static final String TYPE = "gsub";
|
||||||
|
|
||||||
private final List<GsubExpression> gsubExpressions;
|
private final String field;
|
||||||
|
private final Pattern pattern;
|
||||||
|
private final String replacement;
|
||||||
|
|
||||||
GsubProcessor(List<GsubExpression> gsubExpressions) {
|
GsubProcessor(String field, Pattern pattern, String replacement) {
|
||||||
this.gsubExpressions = gsubExpressions;
|
this.field = field;
|
||||||
|
this.pattern = pattern;
|
||||||
|
this.replacement = replacement;
|
||||||
}
|
}
|
||||||
|
|
||||||
List<GsubExpression> getGsubExpressions() {
|
String getField() {
|
||||||
return gsubExpressions;
|
return field;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Pattern getPattern() {
|
||||||
|
return pattern;
|
||||||
|
}
|
||||||
|
|
||||||
|
String getReplacement() {
|
||||||
|
return replacement;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void execute(IngestDocument document) {
|
public void execute(IngestDocument document) {
|
||||||
for (GsubExpression gsubExpression : gsubExpressions) {
|
String oldVal = document.getFieldValue(field, String.class);
|
||||||
String oldVal = document.getFieldValue(gsubExpression.getFieldName(), String.class);
|
if (oldVal == null) {
|
||||||
if (oldVal == null) {
|
throw new IllegalArgumentException("field [" + field + "] is null, cannot match pattern.");
|
||||||
throw new IllegalArgumentException("field [" + gsubExpression.getFieldName() + "] is null, cannot match pattern.");
|
|
||||||
}
|
|
||||||
Matcher matcher = gsubExpression.getPattern().matcher(oldVal);
|
|
||||||
String newVal = matcher.replaceAll(gsubExpression.getReplacement());
|
|
||||||
document.setFieldValue(gsubExpression.getFieldName(), newVal);
|
|
||||||
}
|
}
|
||||||
|
Matcher matcher = pattern.matcher(oldVal);
|
||||||
|
String newVal = matcher.replaceAll(replacement);
|
||||||
|
document.setFieldValue(field, newVal);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -68,25 +77,11 @@ public class GsubProcessor implements Processor {
|
||||||
public static class Factory implements Processor.Factory<GsubProcessor> {
|
public static class Factory implements Processor.Factory<GsubProcessor> {
|
||||||
@Override
|
@Override
|
||||||
public GsubProcessor create(Map<String, Object> config) throws Exception {
|
public GsubProcessor create(Map<String, Object> config) throws Exception {
|
||||||
List<Map<String, String>> gsubConfig = ConfigurationUtils.readList(config, "expressions");
|
String field = ConfigurationUtils.readStringProperty(config, "field");
|
||||||
List<GsubExpression> gsubExpressions = new ArrayList<>();
|
String pattern = ConfigurationUtils.readStringProperty(config, "pattern");
|
||||||
for (Map<String, String> stringObjectMap : gsubConfig) {
|
String replacement = ConfigurationUtils.readStringProperty(config, "replacement");
|
||||||
String field = stringObjectMap.get("field");
|
Pattern searchPattern = Pattern.compile(pattern);
|
||||||
if (field == null) {
|
return new GsubProcessor(field, searchPattern, replacement);
|
||||||
throw new IllegalArgumentException("no [field] specified for gsub expression");
|
|
||||||
}
|
|
||||||
String pattern = stringObjectMap.get("pattern");
|
|
||||||
if (pattern == null) {
|
|
||||||
throw new IllegalArgumentException("no [pattern] specified for gsub expression");
|
|
||||||
}
|
|
||||||
String replacement = stringObjectMap.get("replacement");
|
|
||||||
if (replacement == null) {
|
|
||||||
throw new IllegalArgumentException("no [replacement] specified for gsub expression");
|
|
||||||
}
|
|
||||||
Pattern searchPattern = Pattern.compile(pattern);
|
|
||||||
gsubExpressions.add(new GsubExpression(field, searchPattern, replacement));
|
|
||||||
}
|
|
||||||
return new GsubProcessor(gsubExpressions);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,7 +23,7 @@ import org.elasticsearch.ingest.IngestDocument;
|
||||||
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
||||||
import org.elasticsearch.ingest.processor.Processor;
|
import org.elasticsearch.ingest.processor.Processor;
|
||||||
|
|
||||||
import java.util.Collections;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
@ -36,28 +36,32 @@ public class JoinProcessor implements Processor {
|
||||||
|
|
||||||
public static final String TYPE = "join";
|
public static final String TYPE = "join";
|
||||||
|
|
||||||
private final Map<String, String> fields;
|
private final String field;
|
||||||
|
private final String separator;
|
||||||
|
|
||||||
JoinProcessor(Map<String, String> fields) {
|
JoinProcessor(String field, String separator) {
|
||||||
this.fields = fields;
|
this.field = field;
|
||||||
|
this.separator = separator;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, String> getFields() {
|
String getField() {
|
||||||
return fields;
|
return field;
|
||||||
|
}
|
||||||
|
|
||||||
|
String getSeparator() {
|
||||||
|
return separator;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void execute(IngestDocument document) {
|
public void execute(IngestDocument document) {
|
||||||
for(Map.Entry<String, String> entry : fields.entrySet()) {
|
List<?> list = document.getFieldValue(field, List.class);
|
||||||
List<?> list = document.getFieldValue(entry.getKey(), List.class);
|
if (list == null) {
|
||||||
if (list == null) {
|
throw new IllegalArgumentException("field [" + field + "] is null, cannot join.");
|
||||||
throw new IllegalArgumentException("field [" + entry.getKey() + "] is null, cannot join.");
|
|
||||||
}
|
|
||||||
String joined = list.stream()
|
|
||||||
.map(Object::toString)
|
|
||||||
.collect(Collectors.joining(entry.getValue()));
|
|
||||||
document.setFieldValue(entry.getKey(), joined);
|
|
||||||
}
|
}
|
||||||
|
String joined = list.stream()
|
||||||
|
.map(Object::toString)
|
||||||
|
.collect(Collectors.joining(separator));
|
||||||
|
document.setFieldValue(field, joined);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -68,8 +72,9 @@ public class JoinProcessor implements Processor {
|
||||||
public static class Factory implements Processor.Factory<JoinProcessor> {
|
public static class Factory implements Processor.Factory<JoinProcessor> {
|
||||||
@Override
|
@Override
|
||||||
public JoinProcessor create(Map<String, Object> config) throws Exception {
|
public JoinProcessor create(Map<String, Object> config) throws Exception {
|
||||||
Map<String, String> fields = ConfigurationUtils.readMap(config, "fields");
|
String field = ConfigurationUtils.readStringProperty(config, "field");
|
||||||
return new JoinProcessor(Collections.unmodifiableMap(fields));
|
String separator = ConfigurationUtils.readStringProperty(config, "separator");
|
||||||
|
return new JoinProcessor(field, separator);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,8 +33,8 @@ public class LowercaseProcessor extends AbstractStringProcessor {
|
||||||
|
|
||||||
public static final String TYPE = "lowercase";
|
public static final String TYPE = "lowercase";
|
||||||
|
|
||||||
LowercaseProcessor(Collection<String> fields) {
|
LowercaseProcessor(String field) {
|
||||||
super(fields);
|
super(field);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -49,8 +49,8 @@ public class LowercaseProcessor extends AbstractStringProcessor {
|
||||||
|
|
||||||
public static class Factory extends AbstractStringProcessor.Factory<LowercaseProcessor> {
|
public static class Factory extends AbstractStringProcessor.Factory<LowercaseProcessor> {
|
||||||
@Override
|
@Override
|
||||||
protected LowercaseProcessor newProcessor(Collection<String> fields) {
|
protected LowercaseProcessor newProcessor(String field) {
|
||||||
return new LowercaseProcessor(fields);
|
return new LowercaseProcessor(field);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,9 +23,6 @@ import org.elasticsearch.ingest.IngestDocument;
|
||||||
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
||||||
import org.elasticsearch.ingest.processor.Processor;
|
import org.elasticsearch.ingest.processor.Processor;
|
||||||
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -35,21 +32,19 @@ public class RemoveProcessor implements Processor {
|
||||||
|
|
||||||
public static final String TYPE = "remove";
|
public static final String TYPE = "remove";
|
||||||
|
|
||||||
private final Collection<String> fields;
|
private final String field;
|
||||||
|
|
||||||
RemoveProcessor(Collection<String> fields) {
|
RemoveProcessor(String field) {
|
||||||
this.fields = fields;
|
this.field = field;
|
||||||
}
|
}
|
||||||
|
|
||||||
Collection<String> getFields() {
|
String getField() {
|
||||||
return fields;
|
return field;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void execute(IngestDocument document) {
|
public void execute(IngestDocument document) {
|
||||||
for(String field : fields) {
|
document.removeField(field);
|
||||||
document.removeField(field);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -60,8 +55,8 @@ public class RemoveProcessor implements Processor {
|
||||||
public static class Factory implements Processor.Factory<RemoveProcessor> {
|
public static class Factory implements Processor.Factory<RemoveProcessor> {
|
||||||
@Override
|
@Override
|
||||||
public RemoveProcessor create(Map<String, Object> config) throws Exception {
|
public RemoveProcessor create(Map<String, Object> config) throws Exception {
|
||||||
List<String> fields = ConfigurationUtils.readList(config, "fields");
|
String field = ConfigurationUtils.readStringProperty(config, "field");
|
||||||
return new RemoveProcessor(Collections.unmodifiableList(fields));
|
return new RemoveProcessor(field);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,7 @@ import org.elasticsearch.ingest.IngestDocument;
|
||||||
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
||||||
import org.elasticsearch.ingest.processor.Processor;
|
import org.elasticsearch.ingest.processor.Processor;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
@ -33,37 +34,39 @@ public class RenameProcessor implements Processor {
|
||||||
|
|
||||||
public static final String TYPE = "rename";
|
public static final String TYPE = "rename";
|
||||||
|
|
||||||
private final Map<String, String> fields;
|
private final String oldFieldName;
|
||||||
|
private final String newFieldName;
|
||||||
|
|
||||||
RenameProcessor(Map<String, String> fields) {
|
RenameProcessor(String oldFieldName, String newFieldName) {
|
||||||
this.fields = fields;
|
this.oldFieldName = oldFieldName;
|
||||||
|
this.newFieldName = newFieldName;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, String> getFields() {
|
String getOldFieldName() {
|
||||||
return fields;
|
return oldFieldName;
|
||||||
|
}
|
||||||
|
|
||||||
|
String getNewFieldName() {
|
||||||
|
return newFieldName;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void execute(IngestDocument document) {
|
public void execute(IngestDocument document) {
|
||||||
for(Map.Entry<String, String> entry : fields.entrySet()) {
|
if (document.hasField(oldFieldName) == false) {
|
||||||
String oldFieldName = entry.getKey();
|
throw new IllegalArgumentException("field [" + oldFieldName + "] doesn't exist");
|
||||||
if (document.hasField(oldFieldName) == false) {
|
}
|
||||||
throw new IllegalArgumentException("field [" + oldFieldName + "] doesn't exist");
|
if (document.hasField(newFieldName)) {
|
||||||
}
|
throw new IllegalArgumentException("field [" + newFieldName + "] already exists");
|
||||||
String newFieldName = entry.getValue();
|
}
|
||||||
if (document.hasField(newFieldName)) {
|
|
||||||
throw new IllegalArgumentException("field [" + newFieldName + "] already exists");
|
|
||||||
}
|
|
||||||
|
|
||||||
Object oldValue = document.getFieldValue(entry.getKey(), Object.class);
|
Object oldValue = document.getFieldValue(oldFieldName, Object.class);
|
||||||
document.setFieldValue(newFieldName, oldValue);
|
document.setFieldValue(newFieldName, oldValue);
|
||||||
try {
|
try {
|
||||||
document.removeField(oldFieldName);
|
document.removeField(oldFieldName);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
//remove the new field if the removal of the old one failed
|
//remove the new field if the removal of the old one failed
|
||||||
document.removeField(newFieldName);
|
document.removeField(newFieldName);
|
||||||
throw e;
|
throw e;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -75,8 +78,9 @@ public class RenameProcessor implements Processor {
|
||||||
public static class Factory implements Processor.Factory<RenameProcessor> {
|
public static class Factory implements Processor.Factory<RenameProcessor> {
|
||||||
@Override
|
@Override
|
||||||
public RenameProcessor create(Map<String, Object> config) throws Exception {
|
public RenameProcessor create(Map<String, Object> config) throws Exception {
|
||||||
Map<String, String> fields = ConfigurationUtils.readMap(config, "fields");
|
String field = ConfigurationUtils.readStringProperty(config, "field");
|
||||||
return new RenameProcessor(Collections.unmodifiableMap(fields));
|
String newField = ConfigurationUtils.readStringProperty(config, "to");
|
||||||
|
return new RenameProcessor(field, newField);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,7 @@ import org.elasticsearch.ingest.IngestDocument;
|
||||||
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
import org.elasticsearch.ingest.processor.ConfigurationUtils;
|
||||||
import org.elasticsearch.ingest.processor.Processor;
|
import org.elasticsearch.ingest.processor.Processor;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
@ -34,21 +35,25 @@ public class SetProcessor implements Processor {
|
||||||
|
|
||||||
public static final String TYPE = "set";
|
public static final String TYPE = "set";
|
||||||
|
|
||||||
private final Map<String, Object> fields;
|
private final String field;
|
||||||
|
private final Object value;
|
||||||
|
|
||||||
SetProcessor(Map<String, Object> fields) {
|
SetProcessor(String field, Object value) {
|
||||||
this.fields = fields;
|
this.field = field;
|
||||||
|
this.value = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, Object> getFields() {
|
String getField() {
|
||||||
return fields;
|
return field;
|
||||||
|
}
|
||||||
|
|
||||||
|
Object getValue() {
|
||||||
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void execute(IngestDocument document) {
|
public void execute(IngestDocument document) {
|
||||||
for(Map.Entry<String, Object> entry : fields.entrySet()) {
|
document.setFieldValue(field, value);
|
||||||
document.setFieldValue(entry.getKey(), entry.getValue());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -59,8 +64,9 @@ public class SetProcessor implements Processor {
|
||||||
public static final class Factory implements Processor.Factory<SetProcessor> {
|
public static final class Factory implements Processor.Factory<SetProcessor> {
|
||||||
@Override
|
@Override
|
||||||
public SetProcessor create(Map<String, Object> config) throws Exception {
|
public SetProcessor create(Map<String, Object> config) throws Exception {
|
||||||
Map<String, Object> fields = ConfigurationUtils.readMap(config, "fields");
|
String field = ConfigurationUtils.readStringProperty(config, "field");
|
||||||
return new SetProcessor(Collections.unmodifiableMap(fields));
|
Object value = ConfigurationUtils.readObject(config, "value");
|
||||||
|
return new SetProcessor(field, value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,25 +36,29 @@ public class SplitProcessor implements Processor {
|
||||||
|
|
||||||
public static final String TYPE = "split";
|
public static final String TYPE = "split";
|
||||||
|
|
||||||
private final Map<String, String> fields;
|
private final String field;
|
||||||
|
private final String separator;
|
||||||
|
|
||||||
SplitProcessor(Map<String, String> fields) {
|
SplitProcessor(String field, String separator) {
|
||||||
this.fields = fields;
|
this.field = field;
|
||||||
|
this.separator = separator;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, String> getFields() {
|
String getField() {
|
||||||
return fields;
|
return field;
|
||||||
|
}
|
||||||
|
|
||||||
|
String getSeparator() {
|
||||||
|
return separator;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void execute(IngestDocument document) {
|
public void execute(IngestDocument document) {
|
||||||
for(Map.Entry<String, String> entry : fields.entrySet()) {
|
String oldVal = document.getFieldValue(field, String.class);
|
||||||
String oldVal = document.getFieldValue(entry.getKey(), String.class);
|
if (oldVal == null) {
|
||||||
if (oldVal == null) {
|
throw new IllegalArgumentException("field [" + field + "] is null, cannot split.");
|
||||||
throw new IllegalArgumentException("field [" + entry.getKey() + "] is null, cannot split.");
|
|
||||||
}
|
|
||||||
document.setFieldValue(entry.getKey(), Arrays.asList(oldVal.split(entry.getValue())));
|
|
||||||
}
|
}
|
||||||
|
document.setFieldValue(field, Arrays.asList(oldVal.split(separator)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -65,8 +69,8 @@ public class SplitProcessor implements Processor {
|
||||||
public static class Factory implements Processor.Factory<SplitProcessor> {
|
public static class Factory implements Processor.Factory<SplitProcessor> {
|
||||||
@Override
|
@Override
|
||||||
public SplitProcessor create(Map<String, Object> config) throws Exception {
|
public SplitProcessor create(Map<String, Object> config) throws Exception {
|
||||||
Map<String, String> fields = ConfigurationUtils.readMap(config, "fields");
|
String field = ConfigurationUtils.readStringProperty(config, "field");
|
||||||
return new SplitProcessor(Collections.unmodifiableMap(fields));
|
return new SplitProcessor(field, ConfigurationUtils.readStringProperty(config, "separator"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,8 +31,8 @@ public class TrimProcessor extends AbstractStringProcessor {
|
||||||
|
|
||||||
public static final String TYPE = "trim";
|
public static final String TYPE = "trim";
|
||||||
|
|
||||||
TrimProcessor(Collection<String> fields) {
|
TrimProcessor(String field) {
|
||||||
super(fields);
|
super(field);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -47,8 +47,8 @@ public class TrimProcessor extends AbstractStringProcessor {
|
||||||
|
|
||||||
public static class Factory extends AbstractStringProcessor.Factory<TrimProcessor> {
|
public static class Factory extends AbstractStringProcessor.Factory<TrimProcessor> {
|
||||||
@Override
|
@Override
|
||||||
protected TrimProcessor newProcessor(Collection<String> fields) {
|
protected TrimProcessor newProcessor(String field) {
|
||||||
return new TrimProcessor(fields);
|
return new TrimProcessor(field);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,8 +32,8 @@ public class UppercaseProcessor extends AbstractStringProcessor {
|
||||||
|
|
||||||
public static final String TYPE = "uppercase";
|
public static final String TYPE = "uppercase";
|
||||||
|
|
||||||
UppercaseProcessor(Collection<String> fields) {
|
UppercaseProcessor(String field) {
|
||||||
super(fields);
|
super(field);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -48,8 +48,8 @@ public class UppercaseProcessor extends AbstractStringProcessor {
|
||||||
|
|
||||||
public static class Factory extends AbstractStringProcessor.Factory<UppercaseProcessor> {
|
public static class Factory extends AbstractStringProcessor.Factory<UppercaseProcessor> {
|
||||||
@Override
|
@Override
|
||||||
protected UppercaseProcessor newProcessor(Collection<String> fields) {
|
protected UppercaseProcessor newProcessor(String field) {
|
||||||
return new UppercaseProcessor(fields);
|
return new UppercaseProcessor(field);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,7 +33,7 @@ import static org.hamcrest.Matchers.equalTo;
|
||||||
|
|
||||||
public abstract class AbstractStringProcessorTestCase extends ESTestCase {
|
public abstract class AbstractStringProcessorTestCase extends ESTestCase {
|
||||||
|
|
||||||
protected abstract AbstractStringProcessor newProcessor(Collection<String> fields);
|
protected abstract AbstractStringProcessor newProcessor(String field);
|
||||||
|
|
||||||
protected String modifyInput(String input) {
|
protected String modifyInput(String input) {
|
||||||
return input;
|
return input;
|
||||||
|
@ -43,23 +43,16 @@ public abstract class AbstractStringProcessorTestCase extends ESTestCase {
|
||||||
|
|
||||||
public void testProcessor() throws Exception {
|
public void testProcessor() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
int numFields = randomIntBetween(1, 5);
|
String fieldValue = RandomDocumentPicks.randomString(random());
|
||||||
Map<String, String> expected = new HashMap<>();
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, modifyInput(fieldValue));
|
||||||
for (int i = 0; i < numFields; i++) {
|
Processor processor = newProcessor(fieldName);
|
||||||
String fieldValue = RandomDocumentPicks.randomString(random());
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, modifyInput(fieldValue));
|
|
||||||
expected.put(fieldName, expectedResult(fieldValue));
|
|
||||||
}
|
|
||||||
Processor processor = newProcessor(expected.keySet());
|
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, String> entry : expected.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult(fieldValue)));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), String.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testFieldNotFound() throws Exception {
|
public void testFieldNotFound() throws Exception {
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
Processor processor = newProcessor(Collections.singletonList(fieldName));
|
Processor processor = newProcessor(fieldName);
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
|
@ -70,7 +63,7 @@ public abstract class AbstractStringProcessorTestCase extends ESTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testNullValue() throws Exception {
|
public void testNullValue() throws Exception {
|
||||||
Processor processor = newProcessor(Collections.singletonList("field"));
|
Processor processor = newProcessor("field");
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
|
@ -82,7 +75,7 @@ public abstract class AbstractStringProcessorTestCase extends ESTestCase {
|
||||||
|
|
||||||
public void testNonStringValue() throws Exception {
|
public void testNonStringValue() throws Exception {
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
Processor processor = newProcessor(Collections.singletonList(fieldName));
|
Processor processor = newProcessor(fieldName);
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
ingestDocument.setFieldValue(fieldName, randomInt());
|
ingestDocument.setFieldValue(fieldName, randomInt());
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -34,30 +34,19 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
|
||||||
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
|
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
|
ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
|
||||||
Map<String, String> fields = Collections.singletonMap("field1", type.toString());
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
config.put("type", type.toString());
|
||||||
ConvertProcessor convertProcessor = factory.create(config);
|
ConvertProcessor convertProcessor = factory.create(config);
|
||||||
assertThat(convertProcessor.getFields().size(), equalTo(1));
|
assertThat(convertProcessor.getField(), equalTo("field1"));
|
||||||
assertThat(convertProcessor.getFields().get("field1"), equalTo(type));
|
assertThat(convertProcessor.getConvertType(), equalTo(type));
|
||||||
}
|
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
|
||||||
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
|
|
||||||
Map<String, Object> config = new HashMap<>();
|
|
||||||
try {
|
|
||||||
factory.create(config);
|
|
||||||
fail("factory create should have failed");
|
|
||||||
} catch(IllegalArgumentException e) {
|
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateUnsupportedType() throws Exception {
|
public void testCreateUnsupportedType() throws Exception {
|
||||||
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
|
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
String type = "type-" + randomAsciiOfLengthBetween(1, 10);
|
String type = "type-" + randomAsciiOfLengthBetween(1, 10);
|
||||||
Map<String, String> fields = Collections.singletonMap("field1", type);
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
config.put("type", type);
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
|
@ -65,4 +54,29 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
|
||||||
assertThat(e.getMessage(), Matchers.equalTo("type [" + type + "] not supported, cannot convert field."));
|
assertThat(e.getMessage(), Matchers.equalTo("type [" + type + "] not supported, cannot convert field."));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testCreateNoFieldPresent() throws Exception {
|
||||||
|
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
|
||||||
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
String type = "type-" + randomAsciiOfLengthBetween(1, 10);
|
||||||
|
config.put("type", type);
|
||||||
|
try {
|
||||||
|
factory.create(config);
|
||||||
|
fail("factory create should have failed");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
assertThat(e.getMessage(), Matchers.equalTo("required property [field] is missing"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testCreateNoTypePresent() throws Exception {
|
||||||
|
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
|
||||||
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("field", "field1");
|
||||||
|
try {
|
||||||
|
factory.create(config);
|
||||||
|
fail("factory create should have failed");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
assertThat(e.getMessage(), Matchers.equalTo("required property [type] is missing"));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,45 +34,27 @@ public class ConvertProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testConvertInt() throws Exception {
|
public void testConvertInt() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, ConvertProcessor.Type> fields = new HashMap<>();
|
int randomInt = randomInt();
|
||||||
Map<String, Integer> expectedResult = new HashMap<>();
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt);
|
||||||
int numFields = randomIntBetween(1, 5);
|
Processor processor = new ConvertProcessor(fieldName, Type.INTEGER);
|
||||||
for (int i = 0; i < numFields; i++) {
|
|
||||||
int randomInt = randomInt();
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt);
|
|
||||||
fields.put(fieldName, Type.INTEGER);
|
|
||||||
expectedResult.put(fieldName, randomInt);
|
|
||||||
}
|
|
||||||
Processor processor = new ConvertProcessor(fields);
|
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, Integer> entry : expectedResult.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), Integer.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testConvertIntList() throws Exception {
|
public void testConvertIntList() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, ConvertProcessor.Type> fields = new HashMap<>();
|
int numItems = randomIntBetween(1, 10);
|
||||||
Map<String, List<Integer>> expectedResult = new HashMap<>();
|
List<String> fieldValue = new ArrayList<>();
|
||||||
int numFields = randomIntBetween(1, 5);
|
List<Integer> expectedList = new ArrayList<>();
|
||||||
for (int i = 0; i < numFields; i++) {
|
for (int j = 0; j < numItems; j++) {
|
||||||
int numItems = randomIntBetween(1, 10);
|
int randomInt = randomInt();
|
||||||
List<String> fieldValue = new ArrayList<>();
|
fieldValue.add(Integer.toString(randomInt));
|
||||||
List<Integer> expectedList = new ArrayList<>();
|
expectedList.add(randomInt);
|
||||||
for (int j = 0; j < numItems; j++) {
|
|
||||||
int randomInt = randomInt();
|
|
||||||
fieldValue.add(Integer.toString(randomInt));
|
|
||||||
expectedList.add(randomInt);
|
|
||||||
}
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
|
||||||
fields.put(fieldName, Type.INTEGER);
|
|
||||||
expectedResult.put(fieldName, expectedList);
|
|
||||||
}
|
}
|
||||||
Processor processor = new ConvertProcessor(fields);
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
||||||
|
Processor processor = new ConvertProcessor(fieldName, Type.INTEGER);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, List<Integer>> entry : expectedResult.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), List.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testConvertIntError() throws Exception {
|
public void testConvertIntError() throws Exception {
|
||||||
|
@ -81,8 +63,7 @@ public class ConvertProcessorTests extends ESTestCase {
|
||||||
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
|
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
|
||||||
ingestDocument.setFieldValue(fieldName, value);
|
ingestDocument.setFieldValue(fieldName, value);
|
||||||
|
|
||||||
Map<String, Type> convert = Collections.singletonMap(fieldName, Type.INTEGER);
|
Processor processor = new ConvertProcessor(fieldName, Type.INTEGER);
|
||||||
Processor processor = new ConvertProcessor(convert);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
@ -93,46 +74,30 @@ public class ConvertProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testConvertFloat() throws Exception {
|
public void testConvertFloat() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, Type> fields = new HashMap<>();
|
|
||||||
Map<String, Float> expectedResult = new HashMap<>();
|
Map<String, Float> expectedResult = new HashMap<>();
|
||||||
int numFields = randomIntBetween(1, 5);
|
float randomFloat = randomFloat();
|
||||||
for (int i = 0; i < numFields; i++) {
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat);
|
||||||
float randomFloat = randomFloat();
|
expectedResult.put(fieldName, randomFloat);
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat);
|
|
||||||
fields.put(fieldName, Type.FLOAT);
|
|
||||||
expectedResult.put(fieldName, randomFloat);
|
|
||||||
}
|
|
||||||
|
|
||||||
Processor processor = new ConvertProcessor(fields);
|
Processor processor = new ConvertProcessor(fieldName, Type.FLOAT);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, Float> entry : expectedResult.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), Float.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testConvertFloatList() throws Exception {
|
public void testConvertFloatList() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, Type> fields = new HashMap<>();
|
int numItems = randomIntBetween(1, 10);
|
||||||
Map<String, List<Float>> expectedResult = new HashMap<>();
|
List<String> fieldValue = new ArrayList<>();
|
||||||
int numFields = randomIntBetween(1, 5);
|
List<Float> expectedList = new ArrayList<>();
|
||||||
for (int i = 0; i < numFields; i++) {
|
for (int j = 0; j < numItems; j++) {
|
||||||
int numItems = randomIntBetween(1, 10);
|
float randomFloat = randomFloat();
|
||||||
List<String> fieldValue = new ArrayList<>();
|
fieldValue.add(Float.toString(randomFloat));
|
||||||
List<Float> expectedList = new ArrayList<>();
|
expectedList.add(randomFloat);
|
||||||
for (int j = 0; j < numItems; j++) {
|
|
||||||
float randomFloat = randomFloat();
|
|
||||||
fieldValue.add(Float.toString(randomFloat));
|
|
||||||
expectedList.add(randomFloat);
|
|
||||||
}
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
|
||||||
fields.put(fieldName, Type.FLOAT);
|
|
||||||
expectedResult.put(fieldName, expectedList);
|
|
||||||
}
|
}
|
||||||
Processor processor = new ConvertProcessor(fields);
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
||||||
|
Processor processor = new ConvertProcessor(fieldName, Type.FLOAT);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, List<Float>> entry : expectedResult.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), List.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testConvertFloatError() throws Exception {
|
public void testConvertFloatError() throws Exception {
|
||||||
|
@ -141,8 +106,7 @@ public class ConvertProcessorTests extends ESTestCase {
|
||||||
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
|
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
|
||||||
ingestDocument.setFieldValue(fieldName, value);
|
ingestDocument.setFieldValue(fieldName, value);
|
||||||
|
|
||||||
Map<String, Type> convert = Collections.singletonMap(fieldName, Type.FLOAT);
|
Processor processor = new ConvertProcessor(fieldName, Type.FLOAT);
|
||||||
Processor processor = new ConvertProcessor(convert);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
@ -155,52 +119,36 @@ public class ConvertProcessorTests extends ESTestCase {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, Type> fields = new HashMap<>();
|
Map<String, Type> fields = new HashMap<>();
|
||||||
Map<String, Boolean> expectedResult = new HashMap<>();
|
Map<String, Boolean> expectedResult = new HashMap<>();
|
||||||
int numFields = randomIntBetween(1, 5);
|
boolean randomBoolean = randomBoolean();
|
||||||
for (int i = 0; i < numFields; i++) {
|
String booleanString = Boolean.toString(randomBoolean);
|
||||||
|
if (randomBoolean) {
|
||||||
|
booleanString = booleanString.toUpperCase(Locale.ROOT);
|
||||||
|
}
|
||||||
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString);
|
||||||
|
|
||||||
|
Processor processor = new ConvertProcessor(fieldName, Type.BOOLEAN);
|
||||||
|
processor.execute(ingestDocument);
|
||||||
|
assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testConvertBooleanList() throws Exception {
|
||||||
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
|
int numItems = randomIntBetween(1, 10);
|
||||||
|
List<String> fieldValue = new ArrayList<>();
|
||||||
|
List<Boolean> expectedList = new ArrayList<>();
|
||||||
|
for (int j = 0; j < numItems; j++) {
|
||||||
boolean randomBoolean = randomBoolean();
|
boolean randomBoolean = randomBoolean();
|
||||||
String booleanString = Boolean.toString(randomBoolean);
|
String booleanString = Boolean.toString(randomBoolean);
|
||||||
if (randomBoolean) {
|
if (randomBoolean) {
|
||||||
booleanString = booleanString.toUpperCase(Locale.ROOT);
|
booleanString = booleanString.toUpperCase(Locale.ROOT);
|
||||||
}
|
}
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString);
|
fieldValue.add(booleanString);
|
||||||
fields.put(fieldName, Type.BOOLEAN);
|
expectedList.add(randomBoolean);
|
||||||
expectedResult.put(fieldName, randomBoolean);
|
|
||||||
}
|
}
|
||||||
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
||||||
Processor processor = new ConvertProcessor(fields);
|
Processor processor = new ConvertProcessor(fieldName, Type.BOOLEAN);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, Boolean> entry : expectedResult.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), Boolean.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testConvertBooleanList() throws Exception {
|
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
|
||||||
Map<String, Type> fields = new HashMap<>();
|
|
||||||
Map<String, List<Boolean>> expectedResult = new HashMap<>();
|
|
||||||
int numFields = randomIntBetween(1, 5);
|
|
||||||
for (int i = 0; i < numFields; i++) {
|
|
||||||
int numItems = randomIntBetween(1, 10);
|
|
||||||
List<String> fieldValue = new ArrayList<>();
|
|
||||||
List<Boolean> expectedList = new ArrayList<>();
|
|
||||||
for (int j = 0; j < numItems; j++) {
|
|
||||||
boolean randomBoolean = randomBoolean();
|
|
||||||
String booleanString = Boolean.toString(randomBoolean);
|
|
||||||
if (randomBoolean) {
|
|
||||||
booleanString = booleanString.toUpperCase(Locale.ROOT);
|
|
||||||
}
|
|
||||||
fieldValue.add(booleanString);
|
|
||||||
expectedList.add(randomBoolean);
|
|
||||||
}
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
|
||||||
fields.put(fieldName, Type.BOOLEAN);
|
|
||||||
expectedResult.put(fieldName, expectedList);
|
|
||||||
}
|
|
||||||
Processor processor = new ConvertProcessor(fields);
|
|
||||||
processor.execute(ingestDocument);
|
|
||||||
for (Map.Entry<String, List<Boolean>> entry : expectedResult.entrySet()) {
|
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), List.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testConvertBooleanError() throws Exception {
|
public void testConvertBooleanError() throws Exception {
|
||||||
|
@ -215,8 +163,7 @@ public class ConvertProcessorTests extends ESTestCase {
|
||||||
}
|
}
|
||||||
ingestDocument.setFieldValue(fieldName, fieldValue);
|
ingestDocument.setFieldValue(fieldName, fieldValue);
|
||||||
|
|
||||||
Map<String, Type> convert = Collections.singletonMap(fieldName, Type.BOOLEAN);
|
Processor processor = new ConvertProcessor(fieldName, Type.BOOLEAN);
|
||||||
Processor processor = new ConvertProcessor(convert);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
@ -227,94 +174,75 @@ public class ConvertProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testConvertString() throws Exception {
|
public void testConvertString() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, Type> fields = new HashMap<>();
|
Object fieldValue;
|
||||||
Map<String, String> expectedResult = new HashMap<>();
|
String expectedFieldValue;
|
||||||
int numFields = randomIntBetween(1, 5);
|
switch(randomIntBetween(0, 2)) {
|
||||||
for (int i = 0; i < numFields; i++) {
|
case 0:
|
||||||
Object fieldValue;
|
float randomFloat = randomFloat();
|
||||||
String expectedFieldValue;
|
fieldValue = randomFloat;
|
||||||
switch(randomIntBetween(0, 2)) {
|
expectedFieldValue = Float.toString(randomFloat);
|
||||||
case 0:
|
break;
|
||||||
float randomFloat = randomFloat();
|
case 1:
|
||||||
fieldValue = randomFloat;
|
int randomInt = randomInt();
|
||||||
expectedFieldValue = Float.toString(randomFloat);
|
fieldValue = randomInt;
|
||||||
break;
|
expectedFieldValue = Integer.toString(randomInt);
|
||||||
case 1:
|
break;
|
||||||
int randomInt = randomInt();
|
case 2:
|
||||||
fieldValue = randomInt;
|
boolean randomBoolean = randomBoolean();
|
||||||
expectedFieldValue = Integer.toString(randomInt);
|
fieldValue = randomBoolean;
|
||||||
break;
|
expectedFieldValue = Boolean.toString(randomBoolean);
|
||||||
case 2:
|
break;
|
||||||
boolean randomBoolean = randomBoolean();
|
default:
|
||||||
fieldValue = randomBoolean;
|
throw new UnsupportedOperationException();
|
||||||
expectedFieldValue = Boolean.toString(randomBoolean);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
|
||||||
fields.put(fieldName, Type.STRING);
|
|
||||||
expectedResult.put(fieldName, expectedFieldValue);
|
|
||||||
}
|
}
|
||||||
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
||||||
|
|
||||||
Processor processor = new ConvertProcessor(fields);
|
Processor processor = new ConvertProcessor(fieldName, Type.STRING);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, String> entry : expectedResult.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), String.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testConvertStringList() throws Exception {
|
public void testConvertStringList() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, Type> fields = new HashMap<>();
|
int numItems = randomIntBetween(1, 10);
|
||||||
Map<String, List<String>> expectedResult = new HashMap<>();
|
List<Object> fieldValue = new ArrayList<>();
|
||||||
int numFields = randomIntBetween(1, 5);
|
List<String> expectedList = new ArrayList<>();
|
||||||
for (int i = 0; i < numFields; i++) {
|
for (int j = 0; j < numItems; j++) {
|
||||||
int numItems = randomIntBetween(1, 10);
|
Object randomValue;
|
||||||
List<Object> fieldValue = new ArrayList<>();
|
String randomValueString;
|
||||||
List<String> expectedList = new ArrayList<>();
|
switch(randomIntBetween(0, 2)) {
|
||||||
for (int j = 0; j < numItems; j++) {
|
case 0:
|
||||||
Object randomValue;
|
float randomFloat = randomFloat();
|
||||||
String randomValueString;
|
randomValue = randomFloat;
|
||||||
switch(randomIntBetween(0, 2)) {
|
randomValueString = Float.toString(randomFloat);
|
||||||
case 0:
|
break;
|
||||||
float randomFloat = randomFloat();
|
case 1:
|
||||||
randomValue = randomFloat;
|
int randomInt = randomInt();
|
||||||
randomValueString = Float.toString(randomFloat);
|
randomValue = randomInt;
|
||||||
break;
|
randomValueString = Integer.toString(randomInt);
|
||||||
case 1:
|
break;
|
||||||
int randomInt = randomInt();
|
case 2:
|
||||||
randomValue = randomInt;
|
boolean randomBoolean = randomBoolean();
|
||||||
randomValueString = Integer.toString(randomInt);
|
randomValue = randomBoolean;
|
||||||
break;
|
randomValueString = Boolean.toString(randomBoolean);
|
||||||
case 2:
|
break;
|
||||||
boolean randomBoolean = randomBoolean();
|
default:
|
||||||
randomValue = randomBoolean;
|
throw new UnsupportedOperationException();
|
||||||
randomValueString = Boolean.toString(randomBoolean);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new UnsupportedOperationException();
|
|
||||||
}
|
|
||||||
fieldValue.add(randomValue);
|
|
||||||
expectedList.add(randomValueString);
|
|
||||||
}
|
}
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
fieldValue.add(randomValue);
|
||||||
fields.put(fieldName, Type.STRING);
|
expectedList.add(randomValueString);
|
||||||
expectedResult.put(fieldName, expectedList);
|
|
||||||
}
|
}
|
||||||
Processor processor = new ConvertProcessor(fields);
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
||||||
|
Processor processor = new ConvertProcessor(fieldName, Type.STRING);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, List<String>> entry : expectedResult.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), List.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testConvertNonExistingField() throws Exception {
|
public void testConvertNonExistingField() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
Type type = randomFrom(Type.values());
|
Type type = randomFrom(Type.values());
|
||||||
Map<String, Type> convert = Collections.singletonMap(fieldName, type);
|
Processor processor = new ConvertProcessor(fieldName, type);
|
||||||
Processor processor = new ConvertProcessor(convert);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
@ -326,8 +254,7 @@ public class ConvertProcessorTests extends ESTestCase {
|
||||||
public void testConvertNullField() throws Exception {
|
public void testConvertNullField() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
||||||
Type type = randomFrom(Type.values());
|
Type type = randomFrom(Type.values());
|
||||||
Map<String, Type> convert = Collections.singletonMap("field", type);
|
Processor processor = new ConvertProcessor("field", type);
|
||||||
Processor processor = new ConvertProcessor(convert);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
|
|
@ -33,80 +33,51 @@ public class GsubProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
List<Map<String, String>> expressions = new ArrayList<>();
|
config.put("field", "field1");
|
||||||
Map<String, String> expression = new HashMap<>();
|
config.put("pattern", "\\.");
|
||||||
expression.put("field", "field1");
|
config.put("replacement", "-");
|
||||||
expression.put("pattern", "\\.");
|
|
||||||
expression.put("replacement", "-");
|
|
||||||
expressions.add(expression);
|
|
||||||
config.put("expressions", expressions);
|
|
||||||
GsubProcessor gsubProcessor = factory.create(config);
|
GsubProcessor gsubProcessor = factory.create(config);
|
||||||
assertThat(gsubProcessor.getGsubExpressions().size(), equalTo(1));
|
assertThat(gsubProcessor.getField(), equalTo("field1"));
|
||||||
GsubExpression gsubExpression = gsubProcessor.getGsubExpressions().get(0);
|
assertThat(gsubProcessor.getPattern().toString(), equalTo("\\."));
|
||||||
assertThat(gsubExpression.getFieldName(), equalTo("field1"));
|
assertThat(gsubProcessor.getReplacement(), equalTo("-"));
|
||||||
assertThat(gsubExpression.getPattern().toString(), equalTo("\\."));
|
|
||||||
assertThat(gsubExpression.getReplacement(), equalTo("-"));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testCreateMissingExpressions() throws Exception {
|
|
||||||
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
|
||||||
Map<String, Object> config = new HashMap<>();
|
|
||||||
try {
|
|
||||||
factory.create(config);
|
|
||||||
fail("factory create should have failed");
|
|
||||||
} catch(IllegalArgumentException e) {
|
|
||||||
assertThat(e.getMessage(), equalTo("required property [expressions] is missing"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateNoFieldPresent() throws Exception {
|
public void testCreateNoFieldPresent() throws Exception {
|
||||||
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
List<Map<String, String>> expressions = new ArrayList<>();
|
config.put("pattern", "\\.");
|
||||||
Map<String, String> expression = new HashMap<>();
|
config.put("replacement", "-");
|
||||||
expression.put("pattern", "\\.");
|
|
||||||
expression.put("replacement", "-");
|
|
||||||
expressions.add(expression);
|
|
||||||
config.put("expressions", expressions);
|
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("no [field] specified for gsub expression"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateNoPatternPresent() throws Exception {
|
public void testCreateNoPatternPresent() throws Exception {
|
||||||
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
List<Map<String, String>> expressions = new ArrayList<>();
|
config.put("field", "field1");
|
||||||
Map<String, String> expression = new HashMap<>();
|
config.put("replacement", "-");
|
||||||
expression.put("field", "field1");
|
|
||||||
expression.put("replacement", "-");
|
|
||||||
expressions.add(expression);
|
|
||||||
config.put("expressions", expressions);
|
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("no [pattern] specified for gsub expression"));
|
assertThat(e.getMessage(), equalTo("required property [pattern] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateNoReplacementPresent() throws Exception {
|
public void testCreateNoReplacementPresent() throws Exception {
|
||||||
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
GsubProcessor.Factory factory = new GsubProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
List<Map<String, String>> expressions = new ArrayList<>();
|
config.put("field", "field1");
|
||||||
Map<String, String> expression = new HashMap<>();
|
config.put("pattern", "\\.");
|
||||||
expression.put("field", "field1");
|
|
||||||
expression.put("pattern", "\\.");
|
|
||||||
expressions.add(expression);
|
|
||||||
config.put("expressions", expressions);
|
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("no [replacement] specified for gsub expression"));
|
assertThat(e.getMessage(), equalTo("required property [replacement] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,25 +37,17 @@ public class GsubProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testGsub() throws Exception {
|
public void testGsub() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
int numFields = randomIntBetween(1, 5);
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
|
||||||
List<GsubExpression> expressions = new ArrayList<>();
|
Processor processor = new GsubProcessor(fieldName, Pattern.compile("\\."), "-");
|
||||||
for (int i = 0; i < numFields; i++) {
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
|
|
||||||
expressions.add(new GsubExpression(fieldName, Pattern.compile("\\."), "-"));
|
|
||||||
}
|
|
||||||
Processor processor = new GsubProcessor(expressions);
|
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (GsubExpression expression : expressions) {
|
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo("127-0-0-1"));
|
||||||
assertThat(ingestDocument.getFieldValue(expression.getFieldName(), String.class), equalTo("127-0-0-1"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testGsubNotAStringValue() throws Exception {
|
public void testGsubNotAStringValue() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
ingestDocument.setFieldValue(fieldName, 123);
|
ingestDocument.setFieldValue(fieldName, 123);
|
||||||
List<GsubExpression> gsubExpressions = Collections.singletonList(new GsubExpression(fieldName, Pattern.compile("\\."), "-"));
|
Processor processor = new GsubProcessor(fieldName, Pattern.compile("\\."), "-");
|
||||||
Processor processor = new GsubProcessor(gsubExpressions);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execution should have failed");
|
fail("processor execution should have failed");
|
||||||
|
@ -67,8 +59,7 @@ public class GsubProcessorTests extends ESTestCase {
|
||||||
public void testGsubFieldNotFound() throws Exception {
|
public void testGsubFieldNotFound() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
List<GsubExpression> gsubExpressions = Collections.singletonList(new GsubExpression(fieldName, Pattern.compile("\\."), "-"));
|
Processor processor = new GsubProcessor(fieldName, Pattern.compile("\\."), "-");
|
||||||
Processor processor = new GsubProcessor(gsubExpressions);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execution should have failed");
|
fail("processor execution should have failed");
|
||||||
|
@ -79,8 +70,7 @@ public class GsubProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testGsubNullValue() throws Exception {
|
public void testGsubNullValue() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
||||||
List<GsubExpression> gsubExpressions = Collections.singletonList(new GsubExpression("field", Pattern.compile("\\."), "-"));
|
Processor processor = new GsubProcessor("field", Pattern.compile("\\."), "-");
|
||||||
Processor processor = new GsubProcessor(gsubExpressions);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execution should have failed");
|
fail("processor execution should have failed");
|
||||||
|
|
|
@ -21,7 +21,6 @@ package org.elasticsearch.ingest.processor.join;
|
||||||
|
|
||||||
import org.elasticsearch.test.ESTestCase;
|
import org.elasticsearch.test.ESTestCase;
|
||||||
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
@ -32,20 +31,34 @@ public class JoinProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
JoinProcessor.Factory factory = new JoinProcessor.Factory();
|
JoinProcessor.Factory factory = new JoinProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
Map<String, String> fields = Collections.singletonMap("field1", "-");
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
config.put("separator", "-");
|
||||||
JoinProcessor joinProcessor = factory.create(config);
|
JoinProcessor joinProcessor = factory.create(config);
|
||||||
assertThat(joinProcessor.getFields(), equalTo(fields));
|
assertThat(joinProcessor.getField(), equalTo("field1"));
|
||||||
|
assertThat(joinProcessor.getSeparator(), equalTo("-"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
public void testCreateNoFieldPresent() throws Exception {
|
||||||
JoinProcessor.Factory factory = new JoinProcessor.Factory();
|
JoinProcessor.Factory factory = new JoinProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("separator", "-");
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testCreateNoSeparatorPresent() throws Exception {
|
||||||
|
JoinProcessor.Factory factory = new JoinProcessor.Factory();
|
||||||
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("field", "field1");
|
||||||
|
try {
|
||||||
|
factory.create(config);
|
||||||
|
fail("factory create should have failed");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
assertThat(e.getMessage(), equalTo("required property [separator] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,68 +35,49 @@ public class JoinProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testJoinStrings() throws Exception {
|
public void testJoinStrings() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, String> fields = new HashMap<>();
|
int numItems = randomIntBetween(1, 10);
|
||||||
Map<String, String> expectedResultMap = new HashMap<>();
|
String separator = randomFrom(SEPARATORS);
|
||||||
int numFields = randomIntBetween(1, 5);
|
List<String> fieldValue = new ArrayList<>(numItems);
|
||||||
for (int i = 0; i < numFields; i++) {
|
String expectedResult = "";
|
||||||
int numItems = randomIntBetween(1, 10);
|
for (int j = 0; j < numItems; j++) {
|
||||||
String separator = randomFrom(SEPARATORS);
|
String value = randomAsciiOfLengthBetween(1, 10);
|
||||||
List<String> fieldValue = new ArrayList<>(numItems);
|
fieldValue.add(value);
|
||||||
String expectedResult = "";
|
expectedResult += value;
|
||||||
for (int j = 0; j < numItems; j++) {
|
if (j < numItems - 1) {
|
||||||
String value = randomAsciiOfLengthBetween(1, 10);
|
expectedResult += separator;
|
||||||
fieldValue.add(value);
|
|
||||||
expectedResult += value;
|
|
||||||
if (j < numItems - 1) {
|
|
||||||
expectedResult += separator;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
|
||||||
expectedResultMap.put(fieldName, expectedResult);
|
|
||||||
fields.put(fieldName, separator);
|
|
||||||
}
|
}
|
||||||
Processor processor = new JoinProcessor(fields);
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
||||||
|
Processor processor = new JoinProcessor(fieldName, separator);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, String> entry : expectedResultMap.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), String.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testJoinIntegers() throws Exception {
|
public void testJoinIntegers() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, String> fields = new HashMap<>();
|
int numItems = randomIntBetween(1, 10);
|
||||||
Map<String, String> expectedResultMap = new HashMap<>();
|
String separator = randomFrom(SEPARATORS);
|
||||||
int numFields = randomIntBetween(1, 5);
|
List<Integer> fieldValue = new ArrayList<>(numItems);
|
||||||
for (int i = 0; i < numFields; i++) {
|
String expectedResult = "";
|
||||||
int numItems = randomIntBetween(1, 10);
|
for (int j = 0; j < numItems; j++) {
|
||||||
String separator = randomFrom(SEPARATORS);
|
int value = randomInt();
|
||||||
List<Integer> fieldValue = new ArrayList<>(numItems);
|
fieldValue.add(value);
|
||||||
String expectedResult = "";
|
expectedResult += value;
|
||||||
for (int j = 0; j < numItems; j++) {
|
if (j < numItems - 1) {
|
||||||
int value = randomInt();
|
expectedResult += separator;
|
||||||
fieldValue.add(value);
|
|
||||||
expectedResult += value;
|
|
||||||
if (j < numItems - 1) {
|
|
||||||
expectedResult += separator;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
|
||||||
expectedResultMap.put(fieldName, expectedResult);
|
|
||||||
fields.put(fieldName, separator);
|
|
||||||
}
|
}
|
||||||
Processor processor = new JoinProcessor(fields);
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
|
||||||
|
Processor processor = new JoinProcessor(fieldName, separator);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, String> entry : expectedResultMap.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), String.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testJoinNonListField() throws Exception {
|
public void testJoinNonListField() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
ingestDocument.setFieldValue(fieldName, randomAsciiOfLengthBetween(1, 10));
|
ingestDocument.setFieldValue(fieldName, randomAsciiOfLengthBetween(1, 10));
|
||||||
Map<String, String> join = Collections.singletonMap(fieldName, "-");
|
Processor processor = new JoinProcessor(fieldName, "-");
|
||||||
Processor processor = new JoinProcessor(join);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
|
@ -107,7 +88,7 @@ public class JoinProcessorTests extends ESTestCase {
|
||||||
public void testJoinNonExistingField() throws Exception {
|
public void testJoinNonExistingField() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
Processor processor = new JoinProcessor(Collections.singletonMap(fieldName, "-"));
|
Processor processor = new JoinProcessor(fieldName, "-");
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
|
@ -117,7 +98,7 @@ public class JoinProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testJoinNullValue() throws Exception {
|
public void testJoinNullValue() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
||||||
Processor processor = new JoinProcessor(Collections.singletonMap("field", "-"));
|
Processor processor = new JoinProcessor("field", "-");
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
|
|
|
@ -33,20 +33,19 @@ public class LowercaseProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
|
LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
List<String> fields = Collections.singletonList("field1");
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
|
||||||
LowercaseProcessor uppercaseProcessor = factory.create(config);
|
LowercaseProcessor uppercaseProcessor = factory.create(config);
|
||||||
assertThat(uppercaseProcessor.getFields(), equalTo(fields));
|
assertThat(uppercaseProcessor.getField(), equalTo("field1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
public void testCreateMissingField() throws Exception {
|
||||||
LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
|
LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,8 +27,8 @@ import java.util.Locale;
|
||||||
|
|
||||||
public class LowercaseProcessorTests extends AbstractStringProcessorTestCase {
|
public class LowercaseProcessorTests extends AbstractStringProcessorTestCase {
|
||||||
@Override
|
@Override
|
||||||
protected AbstractStringProcessor newProcessor(Collection<String> fields) {
|
protected AbstractStringProcessor newProcessor(String field) {
|
||||||
return new LowercaseProcessor(fields);
|
return new LowercaseProcessor(field);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -33,20 +33,19 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
RemoveProcessor.Factory factory = new RemoveProcessor.Factory();
|
RemoveProcessor.Factory factory = new RemoveProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
List<String> fields = Collections.singletonList("field1");
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
|
||||||
RemoveProcessor removeProcessor = factory.create(config);
|
RemoveProcessor removeProcessor = factory.create(config);
|
||||||
assertThat(removeProcessor.getFields(), equalTo(fields));
|
assertThat(removeProcessor.getField(), equalTo("field1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
public void testCreateMissingField() throws Exception {
|
||||||
RemoveProcessor.Factory factory = new RemoveProcessor.Factory();
|
RemoveProcessor.Factory factory = new RemoveProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,22 +36,16 @@ public class RemoveProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testRemoveFields() throws Exception {
|
public void testRemoveFields() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
int numFields = randomIntBetween(1, 5);
|
String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
|
||||||
Set<String> fields = new HashSet<>();
|
Processor processor = new RemoveProcessor(field);
|
||||||
for (int i = 0; i < numFields; i++) {
|
|
||||||
fields.add(RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument));
|
|
||||||
}
|
|
||||||
Processor processor = new RemoveProcessor(fields);
|
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (String field : fields) {
|
assertThat(ingestDocument.hasField(field), equalTo(false));
|
||||||
assertThat(ingestDocument.hasField(field), equalTo(false));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRemoveNonExistingField() throws Exception {
|
public void testRemoveNonExistingField() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
Processor processor = new RemoveProcessor(Collections.singletonList(fieldName));
|
Processor processor = new RemoveProcessor(fieldName);
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("remove field should have failed");
|
fail("remove field should have failed");
|
||||||
|
|
|
@ -32,20 +32,34 @@ public class RenameProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
RenameProcessor.Factory factory = new RenameProcessor.Factory();
|
RenameProcessor.Factory factory = new RenameProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
Map<String, String> fields = Collections.singletonMap("field1", "value1");
|
config.put("field", "old_field");
|
||||||
config.put("fields", fields);
|
config.put("to", "new_field");
|
||||||
RenameProcessor renameProcessor = factory.create(config);
|
RenameProcessor renameProcessor = factory.create(config);
|
||||||
assertThat(renameProcessor.getFields(), equalTo(fields));
|
assertThat(renameProcessor.getOldFieldName(), equalTo("old_field"));
|
||||||
|
assertThat(renameProcessor.getNewFieldName(), equalTo("new_field"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
public void testCreateNoFieldPresent() throws Exception {
|
||||||
RenameProcessor.Factory factory = new RenameProcessor.Factory();
|
RenameProcessor.Factory factory = new RenameProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("to", "new_field");
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testCreateNoToPresent() throws Exception {
|
||||||
|
RenameProcessor.Factory factory = new RenameProcessor.Factory();
|
||||||
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("field", "old_field");
|
||||||
|
try {
|
||||||
|
factory.create(config);
|
||||||
|
fail("factory create should have failed");
|
||||||
|
} catch(IllegalArgumentException e) {
|
||||||
|
assertThat(e.getMessage(), equalTo("required property [to] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,26 +34,15 @@ public class RenameProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testRename() throws Exception {
|
public void testRename() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
int numFields = randomIntBetween(1, 5);
|
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
|
||||||
Map<String, String> fields = new HashMap<>();
|
Object fieldValue = ingestDocument.getFieldValue(fieldName, Object.class);
|
||||||
Map<String, Object> newFields = new HashMap<>();
|
String newFieldName;
|
||||||
for (int i = 0; i < numFields; i++) {
|
do {
|
||||||
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
|
newFieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
if (fields.containsKey(fieldName)) {
|
} while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
|
||||||
continue;
|
Processor processor = new RenameProcessor(fieldName, newFieldName);
|
||||||
}
|
|
||||||
String newFieldName;
|
|
||||||
do {
|
|
||||||
newFieldName = RandomDocumentPicks.randomFieldName(random());
|
|
||||||
} while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFields.containsKey(newFieldName));
|
|
||||||
newFields.put(newFieldName, ingestDocument.getFieldValue(fieldName, Object.class));
|
|
||||||
fields.put(fieldName, newFieldName);
|
|
||||||
}
|
|
||||||
Processor processor = new RenameProcessor(fields);
|
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, Object> entry : newFields.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
|
||||||
assertThat(ingestDocument.getFieldValue(entry.getKey(), Object.class), equalTo(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRenameArrayElement() throws Exception {
|
public void testRenameArrayElement() throws Exception {
|
||||||
|
@ -69,7 +58,7 @@ public class RenameProcessorTests extends ESTestCase {
|
||||||
document.put("one", one);
|
document.put("one", one);
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
||||||
|
|
||||||
Processor processor = new RenameProcessor(Collections.singletonMap("list.0", "item"));
|
Processor processor = new RenameProcessor("list.0", "item");
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
Object actualObject = ingestDocument.getSource().get("list");
|
Object actualObject = ingestDocument.getSource().get("list");
|
||||||
assertThat(actualObject, instanceOf(List.class));
|
assertThat(actualObject, instanceOf(List.class));
|
||||||
|
@ -82,7 +71,7 @@ public class RenameProcessorTests extends ESTestCase {
|
||||||
assertThat(actualObject, instanceOf(String.class));
|
assertThat(actualObject, instanceOf(String.class));
|
||||||
assertThat(actualObject, equalTo("item1"));
|
assertThat(actualObject, equalTo("item1"));
|
||||||
|
|
||||||
processor = new RenameProcessor(Collections.singletonMap("list.0", "list.3"));
|
processor = new RenameProcessor("list.0", "list.3");
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
@ -97,7 +86,7 @@ public class RenameProcessorTests extends ESTestCase {
|
||||||
public void testRenameNonExistingField() throws Exception {
|
public void testRenameNonExistingField() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
Processor processor = new RenameProcessor(Collections.singletonMap(fieldName, RandomDocumentPicks.randomFieldName(random())));
|
Processor processor = new RenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()));
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
@ -109,7 +98,7 @@ public class RenameProcessorTests extends ESTestCase {
|
||||||
public void testRenameNewFieldAlreadyExists() throws Exception {
|
public void testRenameNewFieldAlreadyExists() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
|
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
|
||||||
Processor processor = new RenameProcessor(Collections.singletonMap(RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument), fieldName));
|
Processor processor = new RenameProcessor(RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument), fieldName);
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
@ -123,7 +112,7 @@ public class RenameProcessorTests extends ESTestCase {
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
ingestDocument.setFieldValue(fieldName, null);
|
ingestDocument.setFieldValue(fieldName, null);
|
||||||
String newFieldName = RandomDocumentPicks.randomFieldName(random());
|
String newFieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
Processor processor = new RenameProcessor(Collections.singletonMap(fieldName, newFieldName));
|
Processor processor = new RenameProcessor(fieldName, newFieldName);
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
assertThat(ingestDocument.hasField(fieldName), equalTo(false));
|
assertThat(ingestDocument.hasField(fieldName), equalTo(false));
|
||||||
assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
|
assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
|
||||||
|
@ -144,7 +133,7 @@ public class RenameProcessorTests extends ESTestCase {
|
||||||
document.put("list", Collections.singletonList("item"));
|
document.put("list", Collections.singletonList("item"));
|
||||||
|
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
||||||
Processor processor = new RenameProcessor(Collections.singletonMap("list", "new_field"));
|
Processor processor = new RenameProcessor("list", "new_field");
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
@ -169,7 +158,7 @@ public class RenameProcessorTests extends ESTestCase {
|
||||||
document.put("list", Collections.singletonList("item"));
|
document.put("list", Collections.singletonList("item"));
|
||||||
|
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
||||||
Processor processor = new RenameProcessor(Collections.singletonMap("list", "new_field"));
|
Processor processor = new RenameProcessor("list", "new_field");
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
|
|
@ -32,20 +32,47 @@ public class SetProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
SetProcessor.Factory factory = new SetProcessor.Factory();
|
SetProcessor.Factory factory = new SetProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
Map<String, String> fields = Collections.singletonMap("field1", "value1");
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
config.put("value", "value1");
|
||||||
SetProcessor setProcessor = factory.create(config);
|
SetProcessor setProcessor = factory.create(config);
|
||||||
assertThat(setProcessor.getFields(), equalTo(fields));
|
assertThat(setProcessor.getField(), equalTo("field1"));
|
||||||
|
assertThat(setProcessor.getValue(), equalTo("value1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
public void testCreateNoFieldPresent() throws Exception {
|
||||||
SetProcessor.Factory factory = new SetProcessor.Factory();
|
SetProcessor.Factory factory = new SetProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("value", "value1");
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testCreateNoValuePresent() throws Exception {
|
||||||
|
SetProcessor.Factory factory = new SetProcessor.Factory();
|
||||||
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("field", "field1");
|
||||||
|
try {
|
||||||
|
factory.create(config);
|
||||||
|
fail("factory create should have failed");
|
||||||
|
} catch(IllegalArgumentException e) {
|
||||||
|
assertThat(e.getMessage(), equalTo("required property [value] is missing"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testCreateNullValue() throws Exception {
|
||||||
|
SetProcessor.Factory factory = new SetProcessor.Factory();
|
||||||
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("field", "field1");
|
||||||
|
config.put("value", null);
|
||||||
|
try {
|
||||||
|
factory.create(config);
|
||||||
|
fail("factory create should have failed");
|
||||||
|
} catch(IllegalArgumentException e) {
|
||||||
|
assertThat(e.getMessage(), equalTo("required property [value] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,45 +32,30 @@ public class SetProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testSetExistingFields() throws Exception {
|
public void testSetExistingFields() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
int numFields = randomIntBetween(1, 5);
|
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
|
||||||
Map<String, Object> fields = new HashMap<>();
|
Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
|
||||||
for (int i = 0; i < numFields; i++) {
|
Processor processor = new SetProcessor(fieldName, fieldValue);
|
||||||
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
|
|
||||||
Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
|
|
||||||
fields.put(fieldName, fieldValue);
|
|
||||||
}
|
|
||||||
Processor processor = new SetProcessor(fields);
|
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
|
assertThat(ingestDocument.hasField(fieldName), equalTo(true));
|
||||||
for (Map.Entry<String, Object> field : fields.entrySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
|
||||||
assertThat(ingestDocument.hasField(field.getKey()), equalTo(true));
|
|
||||||
assertThat(ingestDocument.getFieldValue(field.getKey(), Object.class), equalTo(field.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testSetNewFields() throws Exception {
|
public void testSetNewFields() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
//used to verify that there are no conflicts between subsequent fields going to be added
|
//used to verify that there are no conflicts between subsequent fields going to be added
|
||||||
IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
int numFields = randomIntBetween(1, 5);
|
Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
|
||||||
Map<String, Object> fields = new HashMap<>();
|
String fieldName = RandomDocumentPicks.addRandomField(random(), testIngestDocument, fieldValue);
|
||||||
for (int i = 0; i < numFields; i++) {
|
Processor processor = new SetProcessor(fieldName, fieldValue);
|
||||||
Object fieldValue = RandomDocumentPicks.randomFieldValue(random());
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), testIngestDocument, fieldValue);
|
|
||||||
fields.put(fieldName, fieldValue);
|
|
||||||
}
|
|
||||||
Processor processor = new SetProcessor(fields);
|
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (Map.Entry<String, Object> field : fields.entrySet()) {
|
assertThat(ingestDocument.hasField(fieldName), equalTo(true));
|
||||||
assertThat(ingestDocument.hasField(field.getKey()), equalTo(true));
|
assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue));
|
||||||
assertThat(ingestDocument.getFieldValue(field.getKey(), Object.class), equalTo(field.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testSetFieldsTypeMismatch() throws Exception {
|
public void testSetFieldsTypeMismatch() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
ingestDocument.setFieldValue("field", "value");
|
ingestDocument.setFieldValue("field", "value");
|
||||||
Processor processor = new SetProcessor(Collections.singletonMap("field.inner", "value"));
|
Processor processor = new SetProcessor("field.inner", "value");
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("processor execute should have failed");
|
fail("processor execute should have failed");
|
||||||
|
|
|
@ -32,20 +32,34 @@ public class SplitProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
SplitProcessor.Factory factory = new SplitProcessor.Factory();
|
SplitProcessor.Factory factory = new SplitProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
Map<String, String> fields = Collections.singletonMap("field1", "\\.");
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
config.put("separator", "\\.");
|
||||||
SplitProcessor splitProcessor = factory.create(config);
|
SplitProcessor splitProcessor = factory.create(config);
|
||||||
assertThat(splitProcessor.getFields(), equalTo(fields));
|
assertThat(splitProcessor.getField(), equalTo("field1"));
|
||||||
|
assertThat(splitProcessor.getSeparator(), equalTo("\\."));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
public void testCreateNoFieldPresent() throws Exception {
|
||||||
SplitProcessor.Factory factory = new SplitProcessor.Factory();
|
SplitProcessor.Factory factory = new SplitProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("separator", "\\.");
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testCreateNoSeparatorPresent() throws Exception {
|
||||||
|
SplitProcessor.Factory factory = new SplitProcessor.Factory();
|
||||||
|
Map<String, Object> config = new HashMap<>();
|
||||||
|
config.put("field", "field1");
|
||||||
|
try {
|
||||||
|
factory.create(config);
|
||||||
|
fail("factory create should have failed");
|
||||||
|
} catch(IllegalArgumentException e) {
|
||||||
|
assertThat(e.getMessage(), equalTo("required property [separator] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,24 +33,16 @@ public class SplitProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testSplit() throws Exception {
|
public void testSplit() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
|
||||||
Map<String, String> fields = new HashMap<>();
|
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
|
||||||
int numFields = randomIntBetween(1, 5);
|
Processor processor = new SplitProcessor(fieldName, "\\.");
|
||||||
for (int i = 0; i < numFields; i++) {
|
|
||||||
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
|
|
||||||
fields.put(fieldName, "\\.");
|
|
||||||
}
|
|
||||||
Processor processor = new SplitProcessor(fields);
|
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
for (String field : fields.keySet()) {
|
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1")));
|
||||||
assertThat(ingestDocument.getFieldValue(field, List.class), equalTo(Arrays.asList("127", "0", "0", "1")));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testSplitFieldNotFound() throws Exception {
|
public void testSplitFieldNotFound() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
Map<String, String> split = Collections.singletonMap(fieldName, "\\.");
|
Processor processor = new SplitProcessor(fieldName, "\\.");
|
||||||
Processor processor = new SplitProcessor(split);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("split processor should have failed");
|
fail("split processor should have failed");
|
||||||
|
@ -61,8 +53,7 @@ public class SplitProcessorTests extends ESTestCase {
|
||||||
|
|
||||||
public void testSplitNullValue() throws Exception {
|
public void testSplitNullValue() throws Exception {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
|
||||||
Map<String, String> split = Collections.singletonMap("field", "\\.");
|
Processor processor = new SplitProcessor("field", "\\.");
|
||||||
Processor processor = new SplitProcessor(split);
|
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("split processor should have failed");
|
fail("split processor should have failed");
|
||||||
|
@ -75,7 +66,7 @@ public class SplitProcessorTests extends ESTestCase {
|
||||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
|
||||||
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
String fieldName = RandomDocumentPicks.randomFieldName(random());
|
||||||
ingestDocument.setFieldValue(fieldName, randomInt());
|
ingestDocument.setFieldValue(fieldName, randomInt());
|
||||||
Processor processor = new SplitProcessor(Collections.singletonMap(fieldName, "\\."));
|
Processor processor = new SplitProcessor(fieldName, "\\.");
|
||||||
try {
|
try {
|
||||||
processor.execute(ingestDocument);
|
processor.execute(ingestDocument);
|
||||||
fail("split processor should have failed");
|
fail("split processor should have failed");
|
||||||
|
|
|
@ -33,20 +33,19 @@ public class TrimProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
TrimProcessor.Factory factory = new TrimProcessor.Factory();
|
TrimProcessor.Factory factory = new TrimProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
List<String> fields = Collections.singletonList("field1");
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
|
||||||
TrimProcessor uppercaseProcessor = factory.create(config);
|
TrimProcessor uppercaseProcessor = factory.create(config);
|
||||||
assertThat(uppercaseProcessor.getFields(), equalTo(fields));
|
assertThat(uppercaseProcessor.getField(), equalTo("field1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
public void testCreateMissingField() throws Exception {
|
||||||
TrimProcessor.Factory factory = new TrimProcessor.Factory();
|
TrimProcessor.Factory factory = new TrimProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,13 +22,11 @@ package org.elasticsearch.ingest.processor.trim;
|
||||||
import org.elasticsearch.ingest.processor.AbstractStringProcessor;
|
import org.elasticsearch.ingest.processor.AbstractStringProcessor;
|
||||||
import org.elasticsearch.ingest.processor.AbstractStringProcessorTestCase;
|
import org.elasticsearch.ingest.processor.AbstractStringProcessorTestCase;
|
||||||
|
|
||||||
import java.util.Collection;
|
|
||||||
|
|
||||||
public class TrimProcessorTests extends AbstractStringProcessorTestCase {
|
public class TrimProcessorTests extends AbstractStringProcessorTestCase {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected AbstractStringProcessor newProcessor(Collection<String> fields) {
|
protected AbstractStringProcessor newProcessor(String field) {
|
||||||
return new TrimProcessor(fields);
|
return new TrimProcessor(field);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -33,20 +33,19 @@ public class UppercaseProcessorFactoryTests extends ESTestCase {
|
||||||
public void testCreate() throws Exception {
|
public void testCreate() throws Exception {
|
||||||
UppercaseProcessor.Factory factory = new UppercaseProcessor.Factory();
|
UppercaseProcessor.Factory factory = new UppercaseProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
List<String> fields = Collections.singletonList("field1");
|
config.put("field", "field1");
|
||||||
config.put("fields", fields);
|
|
||||||
UppercaseProcessor uppercaseProcessor = factory.create(config);
|
UppercaseProcessor uppercaseProcessor = factory.create(config);
|
||||||
assertThat(uppercaseProcessor.getFields(), equalTo(fields));
|
assertThat(uppercaseProcessor.getField(), equalTo("field1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testCreateMissingFields() throws Exception {
|
public void testCreateMissingField() throws Exception {
|
||||||
UppercaseProcessor.Factory factory = new UppercaseProcessor.Factory();
|
UppercaseProcessor.Factory factory = new UppercaseProcessor.Factory();
|
||||||
Map<String, Object> config = new HashMap<>();
|
Map<String, Object> config = new HashMap<>();
|
||||||
try {
|
try {
|
||||||
factory.create(config);
|
factory.create(config);
|
||||||
fail("factory create should have failed");
|
fail("factory create should have failed");
|
||||||
} catch(IllegalArgumentException e) {
|
} catch(IllegalArgumentException e) {
|
||||||
assertThat(e.getMessage(), equalTo("required property [fields] is missing"));
|
assertThat(e.getMessage(), equalTo("required property [field] is missing"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,8 +28,8 @@ import java.util.Locale;
|
||||||
public class UppercaseProcessorTests extends AbstractStringProcessorTestCase {
|
public class UppercaseProcessorTests extends AbstractStringProcessorTestCase {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected AbstractStringProcessor newProcessor(Collection<String> fields) {
|
protected AbstractStringProcessor newProcessor(String field) {
|
||||||
return new UppercaseProcessor(fields);
|
return new UppercaseProcessor(field);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -13,9 +13,8 @@
|
||||||
"processors": [
|
"processors": [
|
||||||
{
|
{
|
||||||
"set" : {
|
"set" : {
|
||||||
"fields" : {
|
"field" : "field2",
|
||||||
"field2": "_value"
|
"value": "_value"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
|
@ -13,76 +13,59 @@
|
||||||
"processors": [
|
"processors": [
|
||||||
{
|
{
|
||||||
"set" : {
|
"set" : {
|
||||||
"fields" : {
|
"field" : "new_field",
|
||||||
"new_field": "new_value"
|
"value": "new_value"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"rename" : {
|
"rename" : {
|
||||||
"fields" : {
|
"field" : "field_to_rename",
|
||||||
"field_to_rename": "renamed_field"
|
"to": "renamed_field"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"remove" : {
|
"remove" : {
|
||||||
"fields" : [
|
"field" : "field_to_remove"
|
||||||
"field_to_remove"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"lowercase" : {
|
"lowercase" : {
|
||||||
"fields" : [
|
"field" : "field_to_lowercase"
|
||||||
"field_to_lowercase"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"uppercase" : {
|
"uppercase" : {
|
||||||
"fields" : [
|
"field" : "field_to_uppercase"
|
||||||
"field_to_uppercase"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"trim" : {
|
"trim" : {
|
||||||
"fields" : [
|
"field" : "field_to_trim"
|
||||||
"field_to_trim"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"split" : {
|
"split" : {
|
||||||
"fields" : {
|
"field" : "field_to_split",
|
||||||
"field_to_split": "-"
|
"separator": "-"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"join" : {
|
"join" : {
|
||||||
"fields" : {
|
"field" : "field_to_join",
|
||||||
"field_to_join": "-"
|
"separator": "-"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"convert" : {
|
"convert" : {
|
||||||
"fields" : {
|
"field" : "field_to_convert",
|
||||||
"field_to_convert": "integer"
|
"type": "integer"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"gsub" : {
|
"gsub" : {
|
||||||
"expressions" : [
|
"field": "field_to_gsub",
|
||||||
{
|
"pattern" : "-",
|
||||||
"field": "field_to_gsub",
|
"replacement" : "."
|
||||||
"pattern" : "-",
|
|
||||||
"replacement" : "."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
|
@ -13,9 +13,8 @@
|
||||||
"processors": [
|
"processors": [
|
||||||
{
|
{
|
||||||
"set" : {
|
"set" : {
|
||||||
"fields" : {
|
"field" : "field2",
|
||||||
"field2" : "_value"
|
"value" : "_value"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
@ -67,9 +66,8 @@
|
||||||
"processors": [
|
"processors": [
|
||||||
{
|
{
|
||||||
"set" : {
|
"set" : {
|
||||||
"fields" : {
|
"field" : "field2",
|
||||||
"field2" : "_value"
|
"value" : "_value"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
@ -130,16 +128,14 @@
|
||||||
"processors": [
|
"processors": [
|
||||||
{
|
{
|
||||||
"set" : {
|
"set" : {
|
||||||
"fields" : {
|
"field" : "field2",
|
||||||
"field2" : "_value"
|
"value" : "_value"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"set" : {
|
"set" : {
|
||||||
"fields" : {
|
"field" : "field3",
|
||||||
"field3" : "third_val"
|
"value" : "third_val"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
@ -182,7 +178,7 @@
|
||||||
"processors": [
|
"processors": [
|
||||||
{
|
{
|
||||||
"uppercase" : {
|
"uppercase" : {
|
||||||
"fields" : ["foo"]
|
"field" : "foo"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
@ -227,14 +223,13 @@
|
||||||
"processors": [
|
"processors": [
|
||||||
{
|
{
|
||||||
"convert" : {
|
"convert" : {
|
||||||
"fields" : {
|
"field" : "foo",
|
||||||
"foo": "integer"
|
"type" : "integer"
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"uppercase" : {
|
"uppercase" : {
|
||||||
"fields" : ["bar"]
|
"field" : "bar"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
Loading…
Reference in New Issue