honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

introduce the JSON Processor (#20128) 

introduce the JSON Processor
This commit is contained in:
Tal Levy 2016-09-09 14:34:32 -07:00 committed by GitHub
parent 4b00cc37a1
commit 9f1f5fdedc
4 changed files with 249 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
docs/reference/ingest/ingest-node.asciidoc
View File

@ -1272,6 +1272,28 @@ Throws an error when the field is not an array.
}
--------------------------------------------------
[[json-processor]]
=== JSON Processor
Converts a JSON string into a structured JSON object.
[[json-options]]
.Json Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to be parsed
| `target_field` | no | `field` | The field to insert the converted structured object into
|======
[source,js]
--------------------------------------------------
{
"json": {
"field": "{\"foo\": 2000}"
}
}
--------------------------------------------------
[[lowercase-processor]]
=== Lowercase Processor
Converts a string to its lowercase equivalent.

82
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.ingest.common;
import com.fasterxml.jackson.core.JsonParseException;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import java.util.Map;
/**
* Processor that serializes a string-valued field into a
* map of maps.
*/
public final class JsonProcessor extends AbstractProcessor {
public static final String TYPE = "json";
private final String field;
private final String targetField;
JsonProcessor(String tag, String field, String targetField) {
super(tag);
this.field = field;
this.targetField = targetField;
}
public String getField() {
return field;
}
public String getTargetField() {
return targetField;
}
@Override
public void execute(IngestDocument document) throws Exception {
String stringValue = document.getFieldValue(field, String.class);
try {
Map<String, Object> mapValue = JsonXContent.jsonXContent.createParser(stringValue).map();
document.setFieldValue(targetField, mapValue);
} catch (JsonParseException e) {
throw new IllegalArgumentException(e);
}
}
@Override
public String getType() {
return TYPE;
}
public static final class Factory implements Processor.Factory {
@Override
public JsonProcessor create(Map<String, Processor.Factory> registry, String processorTag,
Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
return new JsonProcessor(processorTag, field, targetField);
}
}
}

69
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.ingest.common;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.CoreMatchers.equalTo;
public class JsonProcessorFactoryTests extends ESTestCase {
private static final JsonProcessor.Factory FACTORY = new JsonProcessor.Factory();
public void testCreate() throws Exception {
String processorTag = randomAsciiOfLength(10);
String randomField = randomAsciiOfLength(10);
String randomTargetField = randomAsciiOfLength(5);
Map<String, Object> config = new HashMap<>();
config.put("field", randomField);
config.put("target_field", randomTargetField);
JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config);
assertThat(jsonProcessor.getTag(), equalTo(processorTag));
assertThat(jsonProcessor.getField(), equalTo(randomField));
assertThat(jsonProcessor.getTargetField(), equalTo(randomTargetField));
}
public void testCreateWithDefaultTarget() throws Exception {
String processorTag = randomAsciiOfLength(10);
String randomField = randomAsciiOfLength(10);
Map<String, Object> config = new HashMap<>();
config.put("field", randomField);
JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, config);
assertThat(jsonProcessor.getTag(), equalTo(processorTag));
assertThat(jsonProcessor.getField(), equalTo(randomField));
assertThat(jsonProcessor.getTargetField(), equalTo(randomField));
}
public void testCreateWithMissingField() throws Exception {
Map<String, Object> config = new HashMap<>();
String processorTag = randomAsciiOfLength(10);
ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
() -> FACTORY.create(null, processorTag, config));
assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
}
}

76
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.ingest.common;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.test.ESTestCase;
import java.util.HashMap;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
public class JsonProcessorTests extends ESTestCase {
@SuppressWarnings("unchecked")
public void testExecute() throws Exception {
String processorTag = randomAsciiOfLength(3);
String randomField = randomAsciiOfLength(3);
String randomTargetField = randomAsciiOfLength(2);
JsonProcessor jsonProcessor = new JsonProcessor(processorTag, randomField, randomTargetField);
Map<String, Object> document = new HashMap<>();
Map<String, Object> randomJsonMap = RandomDocumentPicks.randomSource(random());
XContentBuilder builder = JsonXContent.contentBuilder().map(randomJsonMap);
String randomJson = XContentHelper.convertToJson(builder.bytes(), false);
document.put(randomField, randomJson);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
jsonProcessor.execute(ingestDocument);
Map<String, Object> jsonified = ingestDocument.getFieldValue(randomTargetField, Map.class);
assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified);
}
public void testInvalidJson() {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field");
Map<String, Object> document = new HashMap<>();
document.put("field", "invalid json");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
assertThat(exception.getMessage(), equalTo("com.fasterxml.jackson.core.JsonParseException: Unrecognized token" +
" 'invalid': was expecting ('true', 'false' or 'null')\n" +
" at [Source: invalid json; line: 1, column: 8]"));
}
public void testFieldMissing() {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field");
Map<String, Object> document = new HashMap<>();
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
assertThat(exception.getMessage(), equalTo("field [field] not present as part of path [field]"));
}
}