honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Accept ingest simulate params as ints or strings (#23885) 

* Allow ingest simulate to parse _id, _index, _type, _routing and _parent as either string or int (#23823)

* Generate data that includes Integer and String type fields for testing document parsing.
This commit is contained in:
Stuart Neivandt 2017-08-03 13:29:21 -05:00 committed by Tal Levy
parent 4f5ce5598e
commit 8ef7438d6c
4 changed files with 98 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
View File

@ -174,16 +174,24 @@ public class SimulatePipelineRequest extends ActionRequest {
}
private static List<IngestDocument> parseDocs(Map<String, Object> config) {
List<Map<String, Object>> docs = ConfigurationUtils.readList(null, null, config, Fields.DOCS);
List<Map<String, Object>> docs =
ConfigurationUtils.readList(null, null, config, Fields.DOCS);
List<IngestDocument> ingestDocumentList = new ArrayList<>();
for (Map<String, Object> dataMap : docs) {
Map<String, Object> document = ConfigurationUtils.readMap(null, null, dataMap, Fields.SOURCE);
IngestDocument ingestDocument = new IngestDocument(ConfigurationUtils.readStringProperty(null, null, dataMap, MetaData.INDEX.getFieldName(), "_index"),
ConfigurationUtils.readStringProperty(null, null, dataMap, MetaData.TYPE.getFieldName(), "_type"),
ConfigurationUtils.readStringProperty(null, null, dataMap, MetaData.ID.getFieldName(), "_id"),
ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.ROUTING.getFieldName()),
ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.PARENT.getFieldName()),
document);
Map<String, Object> document = ConfigurationUtils.readMap(null, null,
dataMap, Fields.SOURCE);
String index = ConfigurationUtils.readStringOrIntProperty(null, null,
dataMap, MetaData.INDEX.getFieldName(), "_index");
String type = ConfigurationUtils.readStringOrIntProperty(null, null,
dataMap, MetaData.TYPE.getFieldName(), "_type");
String id = ConfigurationUtils.readStringOrIntProperty(null, null,
dataMap, MetaData.ID.getFieldName(), "_id");
String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null,
dataMap, MetaData.ROUTING.getFieldName());
String parent = ConfigurationUtils.readOptionalStringOrIntProperty(null, null,
dataMap, MetaData.PARENT.getFieldName());
IngestDocument ingestDocument =
new IngestDocument(index, type, id, routing, parent, document);
ingestDocumentList.add(ingestDocument);
}
return ingestDocumentList;

46
core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
View File

@ -92,6 +92,52 @@ public final class ConfigurationUtils {
value.getClass().getName() + "]");
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type string or int a {@link ElasticsearchParseException} is thrown.
* If the property is missing and no default value has been specified a {@link ElasticsearchParseException} is thrown
*/
public static String readStringOrIntProperty(String processorType, String processorTag,
Map<String, Object> configuration, String propertyName, String defaultValue) {
Object value = configuration.remove(propertyName);
if (value == null && defaultValue != null) {
return defaultValue;
} else if (value == null) {
throw newConfigurationException(processorType, processorTag, propertyName,
"required property is missing");
}
return readStringOrInt(processorType, processorTag, propertyName, value);
}
private static String readStringOrInt(String processorType, String processorTag,
String propertyName, Object value) {
if (value == null) {
return null;
}
if (value instanceof String) {
return (String) value;
} else if (value instanceof Integer) {
return String.valueOf(value);
}
throw newConfigurationException(processorType, processorTag, propertyName,
"property isn't a string or int, but of type [" + value.getClass().getName() + "]");
}
/**
* Returns and removes the specified property from the specified configuration map.
*
* If the property value isn't of type string or int a {@link ElasticsearchParseException} is thrown.
*/
public static String readOptionalStringOrIntProperty(String processorType, String processorTag,
Map<String, Object> configuration, String propertyName) {
Object value = configuration.remove(propertyName);
if (value == null) {
return null;
}
return readStringOrInt(processorType, processorTag, propertyName, value);
}
public static Boolean readBooleanProperty(String processorType, String processorTag, Map<String, Object> configuration,
String propertyName, boolean defaultValue) {
Object value = configuration.remove(propertyName);

29
core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.action.ingest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@ -40,6 +41,8 @@ import static org.elasticsearch.action.ingest.SimulatePipelineRequest.Fields;
import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID;
import static org.elasticsearch.ingest.IngestDocument.MetaData.ID;
import static org.elasticsearch.ingest.IngestDocument.MetaData.INDEX;
import static org.elasticsearch.ingest.IngestDocument.MetaData.PARENT;
import static org.elasticsearch.ingest.IngestDocument.MetaData.ROUTING;
import static org.elasticsearch.ingest.IngestDocument.MetaData.TYPE;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
@ -116,20 +119,24 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
requestContent.put(Fields.DOCS, docs);
for (int i = 0; i < numDocs; i++) {
Map<String, Object> doc = new HashMap<>();
String index = randomAlphaOfLengthBetween(1, 10);
String type = randomAlphaOfLengthBetween(1, 10);
String id = randomAlphaOfLengthBetween(1, 10);
doc.put(INDEX.getFieldName(), index);
doc.put(TYPE.getFieldName(), type);
doc.put(ID.getFieldName(), id);
Map<String, Object> expectedDoc = new HashMap<>();
List<IngestDocument.MetaData> fields = Arrays.asList(INDEX, TYPE, ID, ROUTING, PARENT);
for(IngestDocument.MetaData field : fields) {
if(randomBoolean()) {
String value = randomAlphaOfLengthBetween(1, 10);
doc.put(field.getFieldName(), value);
expectedDoc.put(field.getFieldName(), value);
}
else {
Integer value = randomIntBetween(1, 1000000);
doc.put(field.getFieldName(), value);
expectedDoc.put(field.getFieldName(), String.valueOf(value));
}
}
String fieldName = randomAlphaOfLengthBetween(1, 10);
String fieldValue = randomAlphaOfLengthBetween(1, 10);
doc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
docs.add(doc);
Map<String, Object> expectedDoc = new HashMap<>();
expectedDoc.put(INDEX.getFieldName(), index);
expectedDoc.put(TYPE.getFieldName(), type);
expectedDoc.put(ID.getFieldName(), id);
expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
expectedDocs.add(expectedDoc);
}
@ -172,6 +179,8 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
assertThat(metadataMap.get(INDEX), equalTo(expectedDocument.get(INDEX.getFieldName())));
assertThat(metadataMap.get(TYPE), equalTo(expectedDocument.get(TYPE.getFieldName())));
assertThat(metadataMap.get(ID), equalTo(expectedDocument.get(ID.getFieldName())));
assertThat(metadataMap.get(ROUTING), equalTo(expectedDocument.get(ROUTING.getFieldName())));
assertThat(metadataMap.get(PARENT), equalTo(expectedDocument.get(PARENT.getFieldName())));
assertThat(ingestDocument.getSourceAndMetadata(), equalTo(expectedDocument.get(Fields.SOURCE)));
}

17
core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
View File

@ -54,6 +54,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
Map<String, Object> fizz = new HashMap<>();
fizz.put("buzz", "hello world");
config.put("fizz", fizz);
config.put("num", 1);
}
public void testReadStringProperty() {
@ -93,6 +94,22 @@ public class ConfigurationUtilsTests extends ESTestCase {
assertThat(val, equalTo(Collections.singletonList(2)));
}
public void testReadStringOrIntProperty() {
String val1 = ConfigurationUtils.readStringOrIntProperty(null, null, config, "foo", null);
String val2 = ConfigurationUtils.readStringOrIntProperty(null, null, config, "num", null);
assertThat(val1, equalTo("bar"));
assertThat(val2, equalTo("1"));
}
public void testReadStringOrIntPropertyInvalidType() {
try {
ConfigurationUtils.readStringOrIntProperty(null, null, config, "arr", null);
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo(
"[arr] property isn't a string or int, but of type [java.util.Arrays$ArrayList]"));
}
}
public void testReadProcessors() throws Exception {
Processor processor = mock(Processor.class);
Map<String, Processor.Factory> registry =