honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add ignore_missing option to relevant processors (#20194)

This commit is contained in:
Tal Levy 2016-09-09 12:20:18 -07:00 committed by GitHub
parent f58a33eb90
commit dda32545bb
30 changed files with 432 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
View File

@ -27,7 +27,7 @@ import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;

2
core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
View File

@ -34,7 +34,7 @@ import org.junit.Before;
import java.util.Collections;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;

9
core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java
View File

@ -20,22 +20,13 @@
package org.elasticsearch.action.ingest;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;
public class SimulatePipelineRequestTests extends ESTestCase {

2
core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
View File

@ -30,7 +30,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;

2
core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
View File

@ -27,7 +27,7 @@ import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;

2
core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
View File

@ -34,7 +34,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;

31
core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
View File

@ -33,6 +33,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.both;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.endsWith;
@ -1004,34 +1005,4 @@ public class IngestDocumentTests extends ESTestCase {
}
}
public static void assertIngestDocument(Object a, Object b) {
if (a instanceof Map) {
Map<?, ?> mapA = (Map<?, ?>) a;
Map<?, ?> mapB = (Map<?, ?>) b;
for (Map.Entry<?, ?> entry : mapA.entrySet()) {
if (entry.getValue() instanceof List || entry.getValue() instanceof Map) {
assertIngestDocument(entry.getValue(), mapB.get(entry.getKey()));
}
}
} else if (a instanceof List) {
List<?> listA = (List<?>) a;
List<?> listB = (List<?>) b;
for (int i = 0; i < listA.size(); i++) {
Object value = listA.get(i);
if (value instanceof List || value instanceof Map) {
assertIngestDocument(value, listB.get(i));
}
}
} else if (a instanceof byte[]) {
assertArrayEquals((byte[]) a, (byte[])b);
} else if (a instanceof IngestDocument) {
IngestDocument docA = (IngestDocument) a;
IngestDocument docB = (IngestDocument) b;
assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata());
assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata());
} else {
String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName());
assertThat(msg, a, equalTo(b));
}
}
}

32
docs/reference/ingest/ingest-node.asciidoc
View File

@ -706,10 +706,11 @@ such a case, `target_field` will still be updated with the unconverted field val
.Convert Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field whose value is to be converted
| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place
| `type` | yes | - | The type to convert the existing value to
| Name | Required | Default | Description
| `field` | yes | - | The field whose value is to be converted
| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place
| `type` | yes | - | The type to convert the existing value to
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======
[source,js]
@ -1142,6 +1143,7 @@ Grok expression.
| `patterns` | yes | - | An ordered list of grok expression to match and extract named captures with. Returns on the first expression in the list that matches.
| `pattern_definitions` | no | - | A map of pattern-name and pattern tuples defining custom patterns to be used by the current processor. Patterns matching existing names will override the pre-existing definition.
| `trace_match` | no | false | when true, `_ingest._grok_match_index` will be inserted into your matched document's metadata with the index into the pattern found in `patterns` that matched.
| `ignore_missing` | no | false | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======
Here is an example of using the provided patterns to extract out and name structured fields from a string field in
@ -1278,8 +1280,9 @@ Converts a string to its lowercase equivalent.
.Lowercase Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to make lowercase
| Name | Required | Default | Description
| `field` | yes | - | The field to make lowercase
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======
[source,js]
@ -1320,9 +1323,10 @@ Renames an existing field. If the field doesn't exist or the new name is already
.Rename Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to be renamed
| `target_field` | yes | - | The new name of the field
| Name | Required | Default | Description
| `field` | yes | - | The field to be renamed
| `target_field` | yes | - | The new name of the field
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
|======
[source,js]
@ -1462,8 +1466,9 @@ NOTE: This only works on leading and trailing whitespace.
.Trim Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The string-valued field to trim whitespace from
| Name | Required | Default | Description
| `field` | yes | - | The string-valued field to trim whitespace from
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
|======
[source,js]
@ -1483,8 +1488,9 @@ Converts a string to its uppercase equivalent.
.Uppercase Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to make uppercase
| Name | Required | Default | Description
| `field` | yes | - | The field to make uppercase
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======
[source,js]

29
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
View File

@ -32,22 +32,40 @@ import java.util.Map;
*/
abstract class AbstractStringProcessor extends AbstractProcessor {
private final String field;
private final boolean ignoreMissing;
protected AbstractStringProcessor(String tag, String field) {
protected AbstractStringProcessor(String tag, String field, boolean ignoreMissing) {
super(tag);
this.field = field;
this.ignoreMissing = ignoreMissing;
}
public String getField() {
return field;
}
boolean isIgnoreMissing() {
return ignoreMissing;
}
@Override
public final void execute(IngestDocument document) {
String val = document.getFieldValue(field, String.class);
if (val == null) {
String val;
try {
val = document.getFieldValue(field, String.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing && document.hasField(field) != true) {
return;
}
throw e;
}
if (val == null && ignoreMissing) {
return;
} else if (val == null) {
throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
}
document.setFieldValue(field, process(val));
}
@ -64,9 +82,10 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
public AbstractStringProcessor create(Map<String, Processor.Factory> registry, String tag,
Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
return newProcessor(tag, field);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);
return newProcessor(tag, field, ignoreMissing);
}
protected abstract AbstractStringProcessor newProcessor(String processorTag, String field);
protected abstract AbstractStringProcessor newProcessor(String processorTag, String field, boolean ignoreMissing);
}
}

27
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
View File

@ -114,12 +114,14 @@ public final class ConvertProcessor extends AbstractProcessor {
private final String field;
private final String targetField;
private final Type convertType;
private final boolean ignoreMissing;
ConvertProcessor(String tag, String field, String targetField, Type convertType) {
ConvertProcessor(String tag, String field, String targetField, Type convertType, boolean ignoreMissing) {
super(tag);
this.field = field;
this.targetField = targetField;
this.convertType = convertType;
this.ignoreMissing = ignoreMissing;
}
String getField() {
@ -134,11 +136,27 @@ public final class ConvertProcessor extends AbstractProcessor {
return convertType;
}
boolean isIgnoreMissing() {
return ignoreMissing;
}
@Override
public void execute(IngestDocument document) {
Object oldValue = document.getFieldValue(field, Object.class);
Object oldValue = null;
Object newValue;
if (oldValue == null) {
try {
oldValue = document.getFieldValue(field, Object.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing) {
return;
}
throw e;
}
if (oldValue == null && ignoreMissing) {
return;
} else if (oldValue == null) {
throw new IllegalArgumentException("Field [" + field + "] is null, cannot be converted to type [" + convertType + "]");
}
@ -168,7 +186,8 @@ public final class ConvertProcessor extends AbstractProcessor {
String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
Type convertType = Type.fromString(processorTag, "type", typeProperty);
return new ConvertProcessor(processorTag, field, targetField, convertType);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new ConvertProcessor(processorTag, field, targetField, convertType, ignoreMissing);
}
}
}

36
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
View File

@ -39,21 +39,36 @@ public final class GrokProcessor extends AbstractProcessor {
private final String matchField;
private final Grok grok;
private final boolean traceMatch;
private final boolean ignoreMissing;
public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField) {
this(tag, patternBank, matchPatterns, matchField, false);
}
public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField, boolean traceMatch) {
public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField,
boolean traceMatch, boolean ignoreMissing) {
super(tag);
this.matchField = matchField;
this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch));
this.traceMatch = traceMatch;
this.ignoreMissing = ignoreMissing;
}
@Override
public void execute(IngestDocument ingestDocument) throws Exception {
String fieldValue = ingestDocument.getFieldValue(matchField, String.class);
String fieldValue;
try {
fieldValue = ingestDocument.getFieldValue(matchField, String.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing && ingestDocument.hasField(matchField) != true) {
return;
}
throw e;
}
if (fieldValue == null && ignoreMissing) {
return;
} else if (fieldValue == null) {
throw new IllegalArgumentException("field [" + matchField + "] is null, cannot process it.");
}
Map<String, Object> matches = grok.captures(fieldValue);
if (matches == null) {
throw new IllegalArgumentException("Provided Grok expressions do not match field value: [" + fieldValue + "]");
@ -77,10 +92,14 @@ public final class GrokProcessor extends AbstractProcessor {
return TYPE;
}
public Grok getGrok() {
Grok getGrok() {
return grok;
}
boolean isIgnoreMissing() {
return ignoreMissing;
}
String getMatchField() {
return matchField;
}
@ -128,6 +147,7 @@ public final class GrokProcessor extends AbstractProcessor {
String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
if (matchPatterns.isEmpty()) {
throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
@ -139,7 +159,7 @@ public final class GrokProcessor extends AbstractProcessor {
}
try {
return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch);
return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch, ignoreMissing);
} catch (Exception e) {
throw newConfigurationException(TYPE, processorTag, "patterns",
"Invalid regex pattern found in: " + matchPatterns + ". " + e.getMessage());

8
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
View File

@ -30,8 +30,8 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
public static final String TYPE = "lowercase";
LowercaseProcessor(String processorTag, String field) {
super(processorTag, field);
LowercaseProcessor(String processorTag, String field, boolean ignoreMissing) {
super(processorTag, field, ignoreMissing);
}
@Override
@ -51,8 +51,8 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
}
@Override
protected LowercaseProcessor newProcessor(String tag, String field) {
return new LowercaseProcessor(tag, field);
protected LowercaseProcessor newProcessor(String tag, String field, boolean ignoreMissing) {
return new LowercaseProcessor(tag, field, ignoreMissing);
}
}
}

17
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
View File

@ -35,11 +35,13 @@ public final class RenameProcessor extends AbstractProcessor {
private final String field;
private final String targetField;
private final boolean ignoreMissing;
RenameProcessor(String tag, String field, String targetField) {
RenameProcessor(String tag, String field, String targetField, boolean ignoreMissing) {
super(tag);
this.field = field;
this.targetField = targetField;
this.ignoreMissing = ignoreMissing;
}
String getField() {
@ -50,10 +52,18 @@ public final class RenameProcessor extends AbstractProcessor {
return targetField;
}
boolean isIgnoreMissing() {
return ignoreMissing;
}
@Override
public void execute(IngestDocument document) {
if (document.hasField(field, true) == false) {
throw new IllegalArgumentException("field [" + field + "] doesn't exist");
if (ignoreMissing) {
return;
} else {
throw new IllegalArgumentException("field [" + field + "] doesn't exist");
}
}
// We fail here if the target field point to an array slot that is out of range.
// If we didn't do this then we would fail if we set the value in the target_field
@ -85,7 +95,8 @@ public final class RenameProcessor extends AbstractProcessor {
Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
return new RenameProcessor(processorTag, field, targetField);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new RenameProcessor(processorTag, field, targetField, ignoreMissing);
}
}
}

8
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
View File

@ -27,8 +27,8 @@ public final class TrimProcessor extends AbstractStringProcessor {
public static final String TYPE = "trim";
TrimProcessor(String processorTag, String field) {
super(processorTag, field);
TrimProcessor(String processorTag, String field, boolean ignoreMissing) {
super(processorTag, field, ignoreMissing);
}
@Override
@ -48,8 +48,8 @@ public final class TrimProcessor extends AbstractStringProcessor {
}
@Override
protected TrimProcessor newProcessor(String tag, String field) {
return new TrimProcessor(tag, field);
protected TrimProcessor newProcessor(String tag, String field, boolean ignoreMissing) {
return new TrimProcessor(tag, field, ignoreMissing);
}
}
}

8
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
View File

@ -29,8 +29,8 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
public static final String TYPE = "uppercase";
UppercaseProcessor(String processorTag, String field) {
super(processorTag, field);
UppercaseProcessor(String processorTag, String field, boolean ignoreMissing) {
super(processorTag, field, ignoreMissing);
}
@Override
@ -50,8 +50,8 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
}
@Override
protected UppercaseProcessor newProcessor(String tag, String field) {
return new UppercaseProcessor(tag, field);
protected UppercaseProcessor newProcessor(String tag, String field, boolean ignoreMissing) {
return new UppercaseProcessor(tag, field, ignoreMissing);
}
}
}

64
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
View File

@ -27,12 +27,13 @@ import org.elasticsearch.test.ESTestCase;
import java.util.Collections;
import java.util.HashMap;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
public abstract class AbstractStringProcessorTestCase extends ESTestCase {
protected abstract AbstractStringProcessor newProcessor(String field);
protected abstract AbstractStringProcessor newProcessor(String field, boolean ignoreMissing);
protected String modifyInput(String input) {
return input;
@ -44,45 +45,60 @@ public abstract class AbstractStringProcessorTestCase extends ESTestCase {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String fieldValue = RandomDocumentPicks.randomString(random());
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, modifyInput(fieldValue));
Processor processor = newProcessor(fieldName);
Processor processor = newProcessor(fieldName, randomBoolean());
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult(fieldValue)));
}
public void testFieldNotFound() throws Exception {
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = newProcessor(fieldName);
Processor processor = newProcessor(fieldName, false);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
try {
processor.execute(ingestDocument);
fail("processor should have failed");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
}
Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
}
public void testFieldNotFoundWithIgnoreMissing() throws Exception {
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = newProcessor(fieldName, true);
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testNullValue() throws Exception {
Processor processor = newProcessor("field");
Processor processor = newProcessor("field", false);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
try {
processor.execute(ingestDocument);
fail("processor should have failed");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("field [field] is null, cannot process it."));
}
Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("field [field] is null, cannot process it."));
}
public void testNullValueWithIgnoreMissing() throws Exception {
Processor processor = newProcessor("field", true);
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testNonStringValue() throws Exception {
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = newProcessor(fieldName);
Processor processor = newProcessor(fieldName, false);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
ingestDocument.setFieldValue(fieldName, randomInt());
try {
processor.execute(ingestDocument);
fail("processor should have failed");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("field [" + fieldName +
"] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
}
Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("field [" + fieldName +
"] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
}
public void testNonStringValueWithIgnoreMissing() throws Exception {
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = newProcessor(fieldName, true);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
ingestDocument.setFieldValue(fieldName, randomInt());
Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("field [" + fieldName +
"] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
}
}

19
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
View File

@ -27,6 +27,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
public class ConvertProcessorFactoryTests extends ESTestCase {
@ -43,6 +44,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field1"));
assertThat(convertProcessor.getConvertType(), equalTo(type));
assertThat(convertProcessor.isIgnoreMissing(), is(false));
}
public void testCreateUnsupportedType() throws Exception {
@ -100,5 +102,22 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field2"));
assertThat(convertProcessor.getConvertType(), equalTo(type));
assertThat(convertProcessor.isIgnoreMissing(), is(false));
}
public void testCreateWithIgnoreMissing() throws Exception {
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
Map<String, Object> config = new HashMap<>();
ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
config.put("field", "field1");
config.put("type", type.toString());
config.put("ignore_missing", true);
String processorTag = randomAsciiOfLength(10);
ConvertProcessor convertProcessor = factory.create(null, processorTag, config);
assertThat(convertProcessor.getTag(), equalTo(processorTag));
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field1"));
assertThat(convertProcessor.getConvertType(), equalTo(type));
assertThat(convertProcessor.isIgnoreMissing(), is(true));
}
}

59
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
View File

@ -31,6 +31,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.elasticsearch.ingest.common.ConvertProcessor.Type;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@ -42,7 +43,7 @@ public class ConvertProcessorTests extends ESTestCase {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
int randomInt = randomInt();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
}
@ -58,7 +59,7 @@ public class ConvertProcessorTests extends ESTestCase {
expectedList.add(randomInt);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
@ -69,7 +70,7 @@ public class ConvertProcessorTests extends ESTestCase {
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -85,7 +86,7 @@ public class ConvertProcessorTests extends ESTestCase {
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat);
expectedResult.put(fieldName, randomFloat);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat));
}
@ -101,7 +102,7 @@ public class ConvertProcessorTests extends ESTestCase {
expectedList.add(randomFloat);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
@ -112,7 +113,7 @@ public class ConvertProcessorTests extends ESTestCase {
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -130,7 +131,7 @@ public class ConvertProcessorTests extends ESTestCase {
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean));
}
@ -150,7 +151,7 @@ public class ConvertProcessorTests extends ESTestCase {
expectedList.add(randomBoolean);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
@ -167,7 +168,7 @@ public class ConvertProcessorTests extends ESTestCase {
}
ingestDocument.setFieldValue(fieldName, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -201,7 +202,7 @@ public class ConvertProcessorTests extends ESTestCase {
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue));
}
@ -237,7 +238,7 @@ public class ConvertProcessorTests extends ESTestCase {
expectedList.add(randomValueString);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
@ -246,7 +247,7 @@ public class ConvertProcessorTests extends ESTestCase {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Type type = randomFrom(Type.values());
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -258,7 +259,7 @@ public class ConvertProcessorTests extends ESTestCase {
public void testConvertNullField() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
Type type = randomFrom(Type.values());
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -267,6 +268,25 @@ public class ConvertProcessorTests extends ESTestCase {
}
}
public void testConvertNonExistingFieldWithIgnoreMissing() throws Exception {
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
String fieldName = RandomDocumentPicks.randomFieldName(random());
Type type = randomFrom(Type.values());
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type, true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testConvertNullFieldWithIgnoreMissing() throws Exception {
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
Type type = randomFrom(Type.values());
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type, true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testAutoConvertNotString() throws Exception {
Object randomValue;
switch(randomIntBetween(0, 2)) {
@ -286,7 +306,7 @@ public class ConvertProcessorTests extends ESTestCase {
throw new UnsupportedOperationException();
}
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, sameInstance(randomValue));
@ -295,7 +315,7 @@ public class ConvertProcessorTests extends ESTestCase {
public void testAutoConvertStringNotMatched() throws Exception {
String value = "notAnIntFloatOrBool";
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, sameInstance(value));
@ -306,7 +326,7 @@ public class ConvertProcessorTests extends ESTestCase {
String booleanString = Boolean.toString(randomBoolean);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(),
Collections.singletonMap("field", booleanString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomBoolean));
@ -316,7 +336,7 @@ public class ConvertProcessorTests extends ESTestCase {
int randomInt = randomInt();
String randomString = Integer.toString(randomInt);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomInt));
@ -326,7 +346,7 @@ public class ConvertProcessorTests extends ESTestCase {
float randomFloat = randomFloat();
String randomString = Float.toString(randomFloat);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomFloat));
@ -337,10 +357,9 @@ public class ConvertProcessorTests extends ESTestCase {
int randomInt = randomInt();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, String.valueOf(randomInt));
String targetField = fieldName + randomAsciiOfLength(5);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt)));
assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt));
}
}

2
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
View File

@ -86,7 +86,7 @@ public class DotExpanderProcessorTests extends ESTestCase {
// so because foo is no branch field but a value field the `foo.bar` field can't be expanded
// into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
IngestDocument document = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor("_tag", "foo", "foo.bar");
Processor processor = new RenameProcessor("_tag", "foo", "foo.bar", false);
processor.execute(document);
processor = new DotExpanderProcessor("_tag", null, "foo.bar");
processor.execute(document);

4
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
View File

@ -49,7 +49,7 @@ public class ForEachProcessorTests extends ESTestCase {
);
ForEachProcessor processor = new ForEachProcessor(
"_tag", "values", new UppercaseProcessor("_tag", "_ingest._value")
"_tag", "values", new UppercaseProcessor("_tag", "_ingest._value", false)
);
processor.execute(ingestDocument);
@ -197,7 +197,7 @@ public class ForEachProcessorTests extends ESTestCase {
ForEachProcessor processor = new ForEachProcessor(
"_tag", "values", new CompoundProcessor(false,
Collections.singletonList(new UppercaseProcessor("_tag_upper", "_ingest._value")),
Collections.singletonList(new UppercaseProcessor("_tag_upper", "_ingest._value", false)),
Collections.singletonList(new AppendProcessor("_tag",
ts.compile("errors"), (model) -> (Collections.singletonList("added"))))
));

17
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
View File

@ -27,6 +27,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
public class GrokProcessorFactoryTests extends ESTestCase {
@ -42,6 +43,22 @@ public class GrokProcessorFactoryTests extends ESTestCase {
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getMatchField(), equalTo("_field"));
assertThat(processor.getGrok(), notNullValue());
assertThat(processor.isIgnoreMissing(), is(false));
}
public void testBuildWithIgnoreMissing() throws Exception {
GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
config.put("ignore_missing", true);
String processorTag = randomAsciiOfLength(10);
GrokProcessor processor = factory.create(null, processorTag, config);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getMatchField(), equalTo("_field"));
assertThat(processor.getGrok(), notNullValue());
assertThat(processor.isIgnoreMissing(), is(true));
}
public void testBuildMissingField() throws Exception {

56
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
View File

@ -28,6 +28,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
@ -38,7 +39,7 @@ public class GrokProcessorTests extends ESTestCase {
IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
doc.setFieldValue(fieldName, "1");
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
Collections.singletonList("%{ONE:one}"), fieldName);
Collections.singletonList("%{ONE:one}"), fieldName, false, false);
processor.execute(doc);
assertThat(doc.getFieldValue("one", String.class), equalTo("1"));
}
@ -48,7 +49,7 @@ public class GrokProcessorTests extends ESTestCase {
IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
doc.setFieldValue(fieldName, "23");
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
Collections.singletonList("%{ONE:one}"), fieldName);
Collections.singletonList("%{ONE:one}"), fieldName, false, false);
Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
assertThat(e.getMessage(), equalTo("Provided Grok expressions do not match field value: [23]"));
}
@ -59,17 +60,48 @@ public class GrokProcessorTests extends ESTestCase {
originalDoc.setFieldValue(fieldName, fieldName);
IngestDocument doc = new IngestDocument(originalDoc);
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.emptyMap(),
Collections.singletonList(fieldName), fieldName);
Collections.singletonList(fieldName), fieldName, false, false);
processor.execute(doc);
assertThat(doc, equalTo(originalDoc));
}
public void testNullField() {
String fieldName = RandomDocumentPicks.randomFieldName(random());
IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
doc.setFieldValue(fieldName, null);
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
Collections.singletonList("%{ONE:one}"), fieldName, false, false);
Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
assertThat(e.getMessage(), equalTo("field [" + fieldName + "] is null, cannot process it."));
}
public void testNullFieldWithIgnoreMissing() throws Exception {
String fieldName = RandomDocumentPicks.randomFieldName(random());
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
originalIngestDocument.setFieldValue(fieldName, null);
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
Collections.singletonList("%{ONE:one}"), fieldName, false, true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testNotStringField() {
String fieldName = RandomDocumentPicks.randomFieldName(random());
IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
doc.setFieldValue(fieldName, 1);
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
Collections.singletonList("%{ONE:one}"), fieldName);
Collections.singletonList("%{ONE:one}"), fieldName, false, false);
Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
}
public void testNotStringFieldWithIgnoreMissing() {
String fieldName = RandomDocumentPicks.randomFieldName(random());
IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
doc.setFieldValue(fieldName, 1);
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
Collections.singletonList("%{ONE:one}"), fieldName, false, true);
Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
}
@ -78,11 +110,21 @@ public class GrokProcessorTests extends ESTestCase {
String fieldName = "foo.bar";
IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
Collections.singletonList("%{ONE:one}"), fieldName);
Collections.singletonList("%{ONE:one}"), fieldName, false, false);
Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
assertThat(e.getMessage(), equalTo("field [foo] not present as part of path [foo.bar]"));
}
public void testMissingFieldWithIgnoreMissing() throws Exception {
String fieldName = "foo.bar";
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
Collections.singletonList("%{ONE:one}"), fieldName, false, true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testMultiplePatternsWithMatchReturn() throws Exception {
String fieldName = RandomDocumentPicks.randomFieldName(random());
IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
@ -92,7 +134,7 @@ public class GrokProcessorTests extends ESTestCase {
patternBank.put("TWO", "2");
patternBank.put("THREE", "3");
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), patternBank,
Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName);
Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, false, false);
processor.execute(doc);
assertThat(doc.hasField("one"), equalTo(false));
assertThat(doc.getFieldValue("two", String.class), equalTo("2"));
@ -108,7 +150,7 @@ public class GrokProcessorTests extends ESTestCase {
patternBank.put("TWO", "2");
patternBank.put("THREE", "3");
GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), patternBank,
Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, true);
Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, true, false);
processor.execute(doc);
assertThat(doc.hasField("one"), equalTo(false));
assertThat(doc.getFieldValue("two", String.class), equalTo("2"));

14
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
View File

@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.is;
public class LowercaseProcessorFactoryTests extends ESTestCase {
@ -37,6 +38,19 @@ public class LowercaseProcessorFactoryTests extends ESTestCase {
LowercaseProcessor uppercaseProcessor = (LowercaseProcessor)factory.create(null, processorTag, config);
assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
assertThat(uppercaseProcessor.getField(), equalTo("field1"));
assertThat(uppercaseProcessor.isIgnoreMissing(), is(false));
}
public void testCreateWithIgnoreMissing() throws Exception {
LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "field1");
config.put("ignore_missing", true);
String processorTag = randomAsciiOfLength(10);
LowercaseProcessor uppercaseProcessor = (LowercaseProcessor)factory.create(null, processorTag, config);
assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
assertThat(uppercaseProcessor.getField(), equalTo("field1"));
assertThat(uppercaseProcessor.isIgnoreMissing(), is(true));
}
public void testCreateMissingField() throws Exception {

4
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java
View File

@ -23,8 +23,8 @@ import java.util.Locale;
public class LowercaseProcessorTests extends AbstractStringProcessorTestCase {
@Override
protected AbstractStringProcessor newProcessor(String field) {
return new LowercaseProcessor(randomAsciiOfLength(10), field);
protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing) {
return new LowercaseProcessor(randomAsciiOfLength(10), field, ignoreMissing);
}
@Override

15
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
View File

@ -39,6 +39,21 @@ public class RenameProcessorFactoryTests extends ESTestCase {
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
assertThat(renameProcessor.isIgnoreMissing(), equalTo(false));
}
public void testCreateWithIgnoreMissing() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "old_field");
config.put("target_field", "new_field");
config.put("ignore_missing", true);
String processorTag = randomAsciiOfLength(10);
RenameProcessor renameProcessor = factory.create(null, processorTag, config);
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
}
public void testCreateNoFieldPresent() throws Exception {

34
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
View File

@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.nullValue;
@ -44,7 +45,7 @@ public class RenameProcessorTests extends ESTestCase {
do {
newFieldName = RandomDocumentPicks.randomFieldName(random());
} while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName);
Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
}
@ -62,7 +63,7 @@ public class RenameProcessorTests extends ESTestCase {
document.put("one", one);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "item");
Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "item", false);
processor.execute(ingestDocument);
Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
assertThat(actualObject, instanceOf(List.class));
@ -75,7 +76,7 @@ public class RenameProcessorTests extends ESTestCase {
assertThat(actualObject, instanceOf(String.class));
assertThat(actualObject, equalTo("item1"));
processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "list.3");
processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "list.3", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -90,7 +91,8 @@ public class RenameProcessorTests extends ESTestCase {
public void testRenameNonExistingField() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, RandomDocumentPicks.randomFieldName(random()));
Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName,
RandomDocumentPicks.randomFieldName(random()), false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -99,11 +101,21 @@ public class RenameProcessorTests extends ESTestCase {
}
}
public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName,
RandomDocumentPicks.randomFieldName(random()), true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
}
public void testRenameNewFieldAlreadyExists() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
Processor processor = new RenameProcessor(randomAsciiOfLength(10), RandomDocumentPicks.randomExistingFieldName(
random(), ingestDocument), fieldName);
random(), ingestDocument), fieldName, false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -117,7 +129,7 @@ public class RenameProcessorTests extends ESTestCase {
String fieldName = RandomDocumentPicks.randomFieldName(random());
ingestDocument.setFieldValue(fieldName, null);
String newFieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName);
Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(fieldName), equalTo(false));
assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
@ -137,7 +149,7 @@ public class RenameProcessorTests extends ESTestCase {
source.put("list", Collections.singletonList("item"));
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field");
Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -161,7 +173,7 @@ public class RenameProcessorTests extends ESTestCase {
source.put("list", Collections.singletonList("item"));
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field");
Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
@ -176,12 +188,12 @@ public class RenameProcessorTests extends ESTestCase {
Map<String, Object> source = new HashMap<>();
source.put("foo", "bar");
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor1 = new RenameProcessor(randomAsciiOfLength(10), "foo", "foo.bar");
Processor processor1 = new RenameProcessor(randomAsciiOfLength(10), "foo", "foo.bar", false);
processor1.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
Processor processor2 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar", "foo.bar.baz");
Processor processor2 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar", "foo.bar.baz", false);
processor2.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar",
Collections.singletonMap("baz", "bar"))));
@ -189,7 +201,7 @@ public class RenameProcessorTests extends ESTestCase {
assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
// for fun lets try to restore it (which don't allow today)
Processor processor3 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar.baz", "foo");
Processor processor3 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar.baz", "foo", false);
Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("field [foo] already exists"));
}

14
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
View File

@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.is;
public class TrimProcessorFactoryTests extends ESTestCase {
@ -37,6 +38,19 @@ public class TrimProcessorFactoryTests extends ESTestCase {
TrimProcessor uppercaseProcessor = (TrimProcessor)factory.create(null, processorTag, config);
assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
assertThat(uppercaseProcessor.getField(), equalTo("field1"));
assertThat(uppercaseProcessor.isIgnoreMissing(), is(false));
}
public void testCreateWithIgnoreMissing() throws Exception {
TrimProcessor.Factory factory = new TrimProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "field1");
config.put("ignore_missing", true);
String processorTag = randomAsciiOfLength(10);
TrimProcessor uppercaseProcessor = (TrimProcessor)factory.create(null, processorTag, config);
assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
assertThat(uppercaseProcessor.getField(), equalTo("field1"));
assertThat(uppercaseProcessor.isIgnoreMissing(), is(true));
}
public void testCreateMissingField() throws Exception {

4
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java
View File

@ -22,8 +22,8 @@ package org.elasticsearch.ingest.common;
public class TrimProcessorTests extends AbstractStringProcessorTestCase {
@Override
protected AbstractStringProcessor newProcessor(String field) {
return new TrimProcessor(randomAsciiOfLength(10), field);
protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing) {
return new TrimProcessor(randomAsciiOfLength(10), field, ignoreMissing);
}
@Override

4
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java
View File

@ -24,8 +24,8 @@ import java.util.Locale;
public class UppercaseProcessorTests extends AbstractStringProcessorTestCase {
@Override
protected AbstractStringProcessor newProcessor(String field) {
return new UppercaseProcessor(randomAsciiOfLength(10), field);
protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing) {
return new UppercaseProcessor(randomAsciiOfLength(10), field, ignoreMissing);
}
@Override

67
test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.ingest;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertThat;
public class IngestDocumentMatcher {
/**
* Helper method to assert the equivalence between two IngestDocuments.
*
* @param a first object to compare
* @param b second object to compare
*/
public static void assertIngestDocument(Object a, Object b) {
if (a instanceof Map) {
Map<?, ?> mapA = (Map<?, ?>) a;
Map<?, ?> mapB = (Map<?, ?>) b;
for (Map.Entry<?, ?> entry : mapA.entrySet()) {
if (entry.getValue() instanceof List || entry.getValue() instanceof Map) {
assertIngestDocument(entry.getValue(), mapB.get(entry.getKey()));
}
}
} else if (a instanceof List) {
List<?> listA = (List<?>) a;
List<?> listB = (List<?>) b;
for (int i = 0; i < listA.size(); i++) {
Object value = listA.get(i);
if (value instanceof List || value instanceof Map) {
assertIngestDocument(value, listB.get(i));
}
}
} else if (a instanceof byte[]) {
assertArrayEquals((byte[]) a, (byte[])b);
} else if (a instanceof IngestDocument) {
IngestDocument docA = (IngestDocument) a;
IngestDocument docB = (IngestDocument) b;
assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata());
assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata());
} else {
String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName());
assertThat(msg, a, equalTo(b));
}
}
}