honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add json-processor support for non-map json types (#27335) 

The Json Processor originally only supported parsing field values into Maps even
though the JSON spec specifies that strings, null-values, numbers, booleans, and arrays
are also valid JSON types. This commit enables parsing these values now.

response to #25972.
This commit is contained in:
Tal Levy 2017-11-13 10:28:19 -08:00 committed by GitHub
parent 6d30fd5ac0
commit 5c34533761
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 163 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/reference/ingest/ingest-node.asciidoc
View File

@ -1580,6 +1580,8 @@ Converts a JSON string into a structured JSON object.
| `add_to_root` | no | false | Flag that forces the serialized json to be injected into the top level of the document. `target_field` must not be set when this option is chosen.
|======
All JSON-supported types will be parsed (null, boolean, number, array, object, string).
Suppose you provide this configuration of the `json` processor:
[source,js]

43
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
View File

@ -19,14 +19,24 @@
package org.elasticsearch.ingest.common;
import com.fasterxml.jackson.core.JsonParseException;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.json.JsonXContentParser;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
@ -64,17 +74,36 @@ public final class JsonProcessor extends AbstractProcessor {
@Override
public void execute(IngestDocument document) throws Exception {
String stringValue = document.getFieldValue(field, String.class);
try {
Map<String, Object> mapValue = XContentHelper.convertToMap(JsonXContent.jsonXContent, stringValue, false);
if (addToRoot) {
for (Map.Entry<String, Object> entry : mapValue.entrySet()) {
Object fieldValue = document.getFieldValue(field, Object.class);
BytesReference bytesRef = (fieldValue == null) ? new BytesArray("null") : new BytesArray(fieldValue.toString());
try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, bytesRef)) {
XContentParser.Token token = parser.nextToken();
Object value = null;
if (token == XContentParser.Token.VALUE_NULL) {
value = null;
} else if (token == XContentParser.Token.VALUE_STRING) {
value = parser.text();
} else if (token == XContentParser.Token.VALUE_NUMBER) {
value = parser.numberValue();
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
value = parser.booleanValue();
} else if (token == XContentParser.Token.START_OBJECT) {
value = parser.map();
} else if (token == XContentParser.Token.START_ARRAY) {
value = parser.list();
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
throw new IllegalArgumentException("cannot read binary value");
}
if (addToRoot && (value instanceof Map)) {
for (Map.Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
document.setFieldValue(entry.getKey(), entry.getValue());
}
} else if (addToRoot) {
throw new IllegalArgumentException("cannot add non-map fields to root of document");
} else {
document.setFieldValue(targetField, mapValue);
document.setFieldValue(targetField, value);
}
} catch (ElasticsearchParseException e) {
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
}

93
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
View File

@ -21,15 +21,19 @@ package org.elasticsearch.ingest.common;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.test.ESTestCase;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
public class JsonProcessorTests extends ESTestCase {
@ -44,7 +48,7 @@ public class JsonProcessorTests extends ESTestCase {
Map<String, Object> randomJsonMap = RandomDocumentPicks.randomSource(random());
XContentBuilder builder = JsonXContent.contentBuilder().map(randomJsonMap);
String randomJson = XContentHelper.convertToJson(builder.bytes(), false);
String randomJson = XContentHelper.convertToJson(builder.bytes(), false, XContentType.JSON);
document.put(randomField, randomJson);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
@ -53,16 +57,84 @@ public class JsonProcessorTests extends ESTestCase {
assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified);
}
public void testInvalidJson() {
public void testInvalidValue() {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map<String, Object> document = new HashMap<>();
document.put("field", "invalid json");
document.put("field", "blah blah");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
assertThat(exception.getCause().getCause().getMessage(), equalTo("Unrecognized token"
+ " 'invalid': was expecting ('true', 'false' or 'null')\n"
+ " at [Source: invalid json; line: 1, column: 8]"));
assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'blah': " +
"was expecting ('true', 'false' or 'null')"));
}
public void testByteArray() {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map<String, Object> document = new HashMap<>();
document.put("field", new byte[] { 0, 1 });
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'B': was expecting ('true', 'false' or 'null')"));
}
public void testNull() throws Exception {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map<String, Object> document = new HashMap<>();
document.put("field", null);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
jsonProcessor.execute(ingestDocument);
assertNull(ingestDocument.getFieldValue("target_field", Object.class));
}
public void testBoolean() throws Exception {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map<String, Object> document = new HashMap<>();
boolean value = true;
document.put("field", value);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
jsonProcessor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
}
public void testInteger() throws Exception {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map<String, Object> document = new HashMap<>();
int value = 3;
document.put("field", value);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
jsonProcessor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
}
public void testDouble() throws Exception {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map<String, Object> document = new HashMap<>();
double value = 3.0;
document.put("field", value);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
jsonProcessor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
}
public void testString() throws Exception {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map<String, Object> document = new HashMap<>();
String value = "hello world";
document.put("field", "\"" + value + "\"");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
jsonProcessor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
}
public void testArray() throws Exception {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map<String, Object> document = new HashMap<>();
List<Boolean> value = Arrays.asList(true, true, false);
document.put("field", value.toString());
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
jsonProcessor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
}
public void testFieldMissing() {
@ -96,4 +168,13 @@ public class JsonProcessorTests extends ESTestCase {
assertIngestDocument(ingestDocument, expectedIngestDocument);
}
public void testAddBoolToRoot() {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", true);
Map<String, Object> document = new HashMap<>();
document.put("field", true);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
assertThat(exception.getMessage(), containsString("cannot add non-map fields to root of document"));
}
}

41
modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml
View File

@ -15,7 +15,32 @@ teardown:
"processors": [
{
"json" : {
"field" : "foo"
"field" : "foo_object"
}
},
{
"json" : {
"field" : "foo_array"
}
},
{
"json" : {
"field" : "foo_null"
}
},
{
"json" : {
"field" : "foo_string"
}
},
{
"json" : {
"field" : "foo_number"
}
},
{
"json" : {
"field" : "foo_boolean"
}
}
]
@ -29,7 +54,12 @@ teardown:
id: 1
pipeline: "1"
body: {
foo: "{\"hello\": \"world\"}"
foo_object: "{\"hello\": \"world\"}",
foo_array: "[1, 2, 3]",
foo_null: null,
foo_string: "\"bla bla\"",
foo_number: 3,
foo_boolean: "true"
}
- do:
@ -37,4 +67,9 @@ teardown:
index: test
type: test
id: 1
- match: { _source.foo.hello: "world" }
- match: { _source.foo_object.hello: "world" }
- match: { _source.foo_array.0: 1 }
- match: { _source.foo_string: "bla bla" }
- match: { _source.foo_number: 3 }
- is_true: _source.foo_boolean
- is_false: _source.foo_null