Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add jq expression support in flattenSpec (#4171) 

* add jq expression in the flattenSpec

* more tests

* add benchmark

* fix style

* use JsonNode for both JSONPath and JQ

* clean up

* more clean up

* add documentation

* fix style

* move jackson-jq version to dependencyManagement section. remove commented code

* oops. revert wrong fix

* throw IllegalArgumentException for JQ syntax error

* remove e.printStackTrace() that is forbidden

* touch
This commit is contained in:
Kenji Noguchi 2017-09-12 12:18:34 -07:00 committed by Himanshu
parent 4909c48b0c
commit c0be050242
14 changed files with 273 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/pom.xml
View File

@ -87,7 +87,10 @@
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>net.thisptr</groupId>
<artifactId>jackson-jq</artifactId>
</dependency>
<!-- Tests -->
<dependency>
<groupId>junit</groupId>

3
api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java
View File

@ -114,6 +114,9 @@ public class JSONParseSpec extends ParseSpec
case PATH:
type = JSONPathParser.FieldType.PATH;
break;
case JQ:
type = JSONPathParser.FieldType.JQ;
break;
default:
throw new IllegalArgumentException("Invalid type for field " + druidSpec.getName());
}

5
api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java
View File

@ -69,6 +69,11 @@ public class JSONPathFieldSpec
return new JSONPathFieldSpec(JSONPathFieldType.PATH, name, expr);
}
public static JSONPathFieldSpec createJqField(String name, String expr)
{
return new JSONPathFieldSpec(JSONPathFieldType.JQ, name, expr);
}
public static JSONPathFieldSpec createRootField(String name)
{
return new JSONPathFieldSpec(JSONPathFieldType.ROOT, name, null);

3
api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java
View File

@ -26,7 +26,8 @@ import io.druid.java.util.common.StringUtils;
public enum JSONPathFieldType
{
ROOT,
PATH;
PATH,
JQ;
@JsonValue
@Override

18
api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java
View File

@ -41,6 +41,9 @@ public class JSONPathSpecTest
fields.add(JSONPathFieldSpec.createNestedField("hey0barx", "$.hey[0].barx"));
fields.add(JSONPathFieldSpec.createRootField("timestamp"));
fields.add(JSONPathFieldSpec.createRootField("foo.bar1"));
fields.add(JSONPathFieldSpec.createJqField("foobar1", ".foo.bar1"));
fields.add(JSONPathFieldSpec.createJqField("baz0", ".baz[0]"));
fields.add(JSONPathFieldSpec.createJqField("hey0barx", ".hey[0].barx"));
JSONPathSpec flattenSpec = new JSONPathSpec(true, fields);
@ -55,6 +58,9 @@ public class JSONPathSpecTest
JSONPathFieldSpec hey0barx = serdeFields.get(2);
JSONPathFieldSpec timestamp = serdeFields.get(3);
JSONPathFieldSpec foodotbar1 = serdeFields.get(4);
JSONPathFieldSpec jqFoobar1 = serdeFields.get(5);
JSONPathFieldSpec jqBaz0 = serdeFields.get(6);
JSONPathFieldSpec jqHey0barx = serdeFields.get(7);
Assert.assertEquals(JSONPathFieldType.PATH, foobar1.getType());
Assert.assertEquals("foobar1", foobar1.getName());
@ -68,6 +74,18 @@ public class JSONPathSpecTest
Assert.assertEquals("hey0barx", hey0barx.getName());
Assert.assertEquals("$.hey[0].barx", hey0barx.getExpr());
Assert.assertEquals(JSONPathFieldType.JQ, jqFoobar1.getType());
Assert.assertEquals("foobar1", jqFoobar1.getName());
Assert.assertEquals(".foo.bar1", jqFoobar1.getExpr());
Assert.assertEquals(JSONPathFieldType.JQ, jqBaz0.getType());
Assert.assertEquals("baz0", jqBaz0.getName());
Assert.assertEquals(".baz[0]", jqBaz0.getExpr());
Assert.assertEquals(JSONPathFieldType.JQ, jqHey0barx.getType());
Assert.assertEquals("hey0barx", jqHey0barx.getName());
Assert.assertEquals(".hey[0].barx", jqHey0barx.getExpr());
Assert.assertEquals(JSONPathFieldType.ROOT, timestamp.getType());
Assert.assertEquals("timestamp", timestamp.getName());
Assert.assertEquals(null, timestamp.getExpr());

18
benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java
View File

@ -45,12 +45,15 @@ public class FlattenJSONBenchmark
List<String> flatInputs;
List<String> nestedInputs;
List<String> jqInputs;
Parser flatParser;
Parser nestedParser;
Parser jqParser;
Parser fieldDiscoveryParser;
Parser forcedPathParser;
int flatCounter = 0;
int nestedCounter = 0;
int jqCounter = 0;
@Setup
public void prepare() throws Exception
@ -64,9 +67,14 @@ public class FlattenJSONBenchmark
for (int i = 0; i < numEvents; i++) {
nestedInputs.add(gen.generateNestedEvent());
}
jqInputs = new ArrayList<String>();
for (int i = 0; i < numEvents; i++) {
jqInputs.add(gen.generateNestedEvent()); // reuse the same event as "nested"
}
flatParser = gen.getFlatParser();
nestedParser = gen.getNestedParser();
jqParser = gen.getJqParser();
fieldDiscoveryParser = gen.getFieldDiscoveryParser();
forcedPathParser = gen.getForcedPathParser();
}
@ -91,6 +99,16 @@ public class FlattenJSONBenchmark
return parsed;
}
@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public Map<String, Object> jqflatten()
{
Map<String, Object> parsed = jqParser.parse(jqInputs.get(jqCounter));
jqCounter = (jqCounter + 1) % numEvents;
return parsed;
}
@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)

40
benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java
View File

@ -164,6 +164,46 @@ public class FlattenJSONBenchmarkUtil
return spec.makeParser();
}
public Parser getJqParser()
{
List<JSONPathFieldSpec> fields = new ArrayList<>();
fields.add(JSONPathFieldSpec.createRootField("ts"));
fields.add(JSONPathFieldSpec.createRootField("d1"));
fields.add(JSONPathFieldSpec.createJqField("e1.d1", ".e1.d1"));
fields.add(JSONPathFieldSpec.createJqField("e1.d2", ".e1.d2"));
fields.add(JSONPathFieldSpec.createJqField("e2.d3", ".e2.d3"));
fields.add(JSONPathFieldSpec.createJqField("e2.d4", ".e2.d4"));
fields.add(JSONPathFieldSpec.createJqField("e2.d5", ".e2.d5"));
fields.add(JSONPathFieldSpec.createJqField("e2.d6", ".e2.d6"));
fields.add(JSONPathFieldSpec.createJqField("e2.ad1[0]", ".e2.ad1[0]"));
fields.add(JSONPathFieldSpec.createJqField("e2.ad1[1]", ".e2.ad1[1]"));
fields.add(JSONPathFieldSpec.createJqField("e2.ad1[2]", ".e2.ad1[2]"));
fields.add(JSONPathFieldSpec.createJqField("ae1[0].d1", ".ae1[0].d1"));
fields.add(JSONPathFieldSpec.createJqField("ae1[1].d1", ".ae1[1].d1"));
fields.add(JSONPathFieldSpec.createJqField("ae1[2].e1.d2", ".ae1[2].e1.d2"));
fields.add(JSONPathFieldSpec.createRootField("m3"));
fields.add(JSONPathFieldSpec.createJqField("e3.m1", ".e3.m1"));
fields.add(JSONPathFieldSpec.createJqField("e3.m2", ".e3.m2"));
fields.add(JSONPathFieldSpec.createJqField("e3.m3", ".e3.m3"));
fields.add(JSONPathFieldSpec.createJqField("e3.m4", ".e3.m4"));
fields.add(JSONPathFieldSpec.createJqField("e3.am1[0]", ".e3.am1[0]"));
fields.add(JSONPathFieldSpec.createJqField("e3.am1[1]", ".e3.am1[1]"));
fields.add(JSONPathFieldSpec.createJqField("e3.am1[2]", ".e3.am1[2]"));
fields.add(JSONPathFieldSpec.createJqField("e3.am1[3]", ".e3.am1[3]"));
fields.add(JSONPathFieldSpec.createJqField("e4.e4.m4", ".e4.e4.m4"));
JSONPathSpec flattenSpec = new JSONPathSpec(true, fields);
JSONParseSpec spec = new JSONParseSpec(
new TimestampSpec("ts", "iso", null),
new DimensionsSpec(null, null, null),
flattenSpec,
null
);
return spec.makeParser();
}
public String generateFlatEvent() throws Exception
{

3
benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java
View File

@ -38,12 +38,15 @@ public class FlattenJSONBenchmarkUtilTest
Parser flatParser = eventGen.getFlatParser();
Parser nestedParser = eventGen.getNestedParser();
Parser jqParser = eventGen.getJqParser();
Map<String, Object> event = flatParser.parse(newEvent);
Map<String, Object> event2 = nestedParser.parse(newEvent2);
Map<String, Object> event3 = jqParser.parse(newEvent2); // reuse the same event as "nested"
checkEvent1(event);
checkEvent2(event2);
checkEvent2(event3); // make sure JQ parser output matches with JSONPath parser output
}
public void checkEvent1(Map<String, Object> event)

15
docs/content/ingestion/flatten-json.md
View File

@ -17,9 +17,9 @@ Defining the JSON Flatten Spec allows nested JSON fields to be flattened during
| Field | Type | Description | Required |
|-------|------|-------------|----------|
| type | String | Type of the field, "root" or "path". | yes |
| type | String | Type of the field, "root", "path" or "jq". | yes |
| name | String | This string will be used as the column name when the data has been ingested. | yes |
| expr | String | Defines an expression for accessing the field within the JSON object, using [JsonPath](https://github.com/jayway/JsonPath) notation. Only used for type "path", otherwise ignored. | only for type "path" |
| expr | String | Defines an expression for accessing the field within the JSON object, using [JsonPath](https://github.com/jayway/JsonPath) notation for type "path", and [jackson-jq](https://github.com/eiiches/jackson-jq) for type "jq". This field is only used for type "path" and "jq", otherwise ignored. | only for type "path" or "jq" |
Suppose the event JSON has the following form:
@ -99,6 +99,16 @@ To flatten this JSON, the parseSpec could be defined as follows:
"type": "path",
"name": "second-food",
"expr": "$.thing.food[1]"
},
{
"type": "jq",
"name": "first-food-by-jq",
"expr": ".thing.food[1]"
},
{
"type": "jq",
"name": "hello-total",
"expr": ".hello | sum"
}
]
},
@ -147,3 +157,4 @@ Note that:
* If auto field discovery is enabled, any discovered field with the same name as one already defined in the field specs will be skipped and not added twice.
* The JSON input must be a JSON object at the root, not an array. e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported but [{"invalid": "true"}] and [1,2,3] are not.
* [http://jsonpath.herokuapp.com/](http://jsonpath.herokuapp.com/) is useful for testing the path expressions.
* jackson-jq supports subset of [./jq](https://stedolan.github.io/jq/) syntax. Please refer jackson-jq document.

4
java-util/pom.xml
View File

@ -107,6 +107,10 @@
<scope>test</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>net.thisptr</groupId>
<artifactId>jackson-jq</artifactId>
</dependency>
</dependencies>
<build>

60
java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.java.util.common.parsers;
import com.fasterxml.jackson.databind.JsonNode;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
import net.thisptr.jackson.jq.JsonQuery;
import net.thisptr.jackson.jq.exception.JsonQueryException;
public class FlattenExpr
{
private JsonPath jsonPathExpr;
private JsonQuery jsonQueryExpr;
FlattenExpr(JsonPath jsonPathExpr)
{
this.jsonPathExpr = jsonPathExpr;
}
FlattenExpr(JsonQuery jsonQueryExpr)
{
this.jsonQueryExpr = jsonQueryExpr;
}
public JsonNode readPath(JsonNode document, Configuration jsonConfig)
{
return this.jsonPathExpr.read(document, jsonConfig);
}
public JsonNode readJq(JsonNode document)
{
try {
return this.jsonQueryExpr.apply(document).get(0);
}
catch (JsonQueryException e) {
// ignore errors.
}
return null;
}
}

115
java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java
View File

@ -19,21 +19,23 @@
package io.druid.java.util.common.parsers;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider;
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
import io.druid.java.util.common.Pair;
import io.druid.java.util.common.StringUtils;
import net.thisptr.jackson.jq.JsonQuery;
import net.thisptr.jackson.jq.exception.JsonQueryException;
import java.math.BigInteger;
import java.nio.charset.CharsetEncoder;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -43,7 +45,7 @@ import java.util.Map;
*/
public class JSONPathParser implements Parser<String, Object>
{
private final Map<String, Pair<FieldType, JsonPath>> fieldPathMap;
private final Map<String, Pair<FieldType, FlattenExpr>> fieldPathMap;
private final boolean useFieldDiscovery;
private final ObjectMapper mapper;
private final CharsetEncoder enc = Charsets.UTF_8.newEncoder();
@ -65,7 +67,7 @@ public class JSONPathParser implements Parser<String, Object>
// Avoid using defaultConfiguration, as this depends on json-smart which we are excluding.
this.jsonPathConfig = Configuration.builder()
.jsonProvider(new JacksonJsonProvider())
.jsonProvider(new JacksonJsonNodeJsonProvider())
.mappingProvider(new JacksonMappingProvider())
.options(EnumSet.of(Option.SUPPRESS_EXCEPTIONS))
.build();
@ -94,27 +96,26 @@ public class JSONPathParser implements Parser<String, Object>
{
try {
Map<String, Object> map = new LinkedHashMap<>();
Map<String, Object> document = mapper.readValue(
input,
new TypeReference<Map<String, Object>>()
{
}
);
for (Map.Entry<String, Pair<FieldType, JsonPath>> entry : fieldPathMap.entrySet()) {
JsonNode document = mapper.readValue(input, JsonNode.class);
for (Map.Entry<String, Pair<FieldType, FlattenExpr>> entry : fieldPathMap.entrySet()) {
String fieldName = entry.getKey();
Pair<FieldType, JsonPath> pair = entry.getValue();
JsonPath path = pair.rhs;
Object parsedVal;
Pair<FieldType, FlattenExpr> pair = entry.getValue();
FlattenExpr path = pair.rhs;
JsonNode parsedVal;
if (pair.lhs == FieldType.ROOT) {
parsedVal = document.get(fieldName);
} else if (pair.lhs == FieldType.PATH) {
parsedVal = path.readPath(document, jsonPathConfig);
} else if (pair.lhs == FieldType.JQ) {
parsedVal = path.readJq(document);
} else {
parsedVal = path.read(document, jsonPathConfig);
throw new ParseException("Unknown FieldType", pair.lhs);
}
if (parsedVal == null) {
continue;
}
parsedVal = valueConversionFunction(parsedVal);
map.put(fieldName, parsedVal);
map.put(fieldName, valueConversionFunction(parsedVal));
}
if (useFieldDiscovery) {
discoverFields(map, document);
@ -126,70 +127,85 @@ public class JSONPathParser implements Parser<String, Object>
}
}
private Map<String, Pair<FieldType, JsonPath>> generateFieldPaths(List<FieldSpec> fieldSpecs)
private Map<String, Pair<FieldType, FlattenExpr>> generateFieldPaths(List<FieldSpec> fieldSpecs)
{
Map<String, Pair<FieldType, JsonPath>> map = new LinkedHashMap<>();
Map<String, Pair<FieldType, FlattenExpr>> map = new LinkedHashMap<>();
for (FieldSpec fieldSpec : fieldSpecs) {
String fieldName = fieldSpec.getName();
if (map.get(fieldName) != null) {
throw new IllegalArgumentException("Cannot have duplicate field definition: " + fieldName);
}
JsonPath path = fieldSpec.getType() == FieldType.PATH ? JsonPath.compile(fieldSpec.getExpr()) : null;
Pair<FieldType, JsonPath> pair = new Pair<>(fieldSpec.getType(), path);
FlattenExpr path = null;
if (fieldSpec.getType() == FieldType.PATH) {
path = new FlattenExpr(JsonPath.compile(fieldSpec.getExpr()));
} else if (fieldSpec.getType() == FieldType.JQ) {
try {
path = new FlattenExpr(JsonQuery.compile(fieldSpec.getExpr()));
}
catch (JsonQueryException e) {
throw new IllegalArgumentException("Unable to compile JQ expression: " + fieldSpec.getExpr());
}
}
Pair<FieldType, FlattenExpr> pair = new Pair<>(fieldSpec.getType(), path);
map.put(fieldName, pair);
}
return map;
}
private void discoverFields(Map<String, Object> map, Map<String, Object> document)
private void discoverFields(Map<String, Object> map, JsonNode document)
{
for (Map.Entry<String, Object> e : document.entrySet()) {
for (Iterator<Map.Entry<String, JsonNode>> it = document.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> e = it.next();
String field = e.getKey();
if (!map.containsKey(field)) {
Object val = e.getValue();
if (val == null) {
JsonNode val = e.getValue();
if (val.isNull()) {
continue;
}
if (val instanceof Map) {
if (val.isObject()) {
continue;
}
if (val instanceof List) {
if (!isFlatList((List) val)) {
if (val.isArray()) {
if (!isFlatList(val)) {
continue;
}
}
val = valueConversionFunction(val);
map.put(field, val);
map.put(field, valueConversionFunction(val));
}
}
}
private Object valueConversionFunction(Object val)
private Object valueConversionFunction(JsonNode val)
{
if (val instanceof Integer) {
return Long.valueOf((Integer) val);
if (val == null) {
return null;
}
if (val instanceof BigInteger) {
return Double.valueOf(((BigInteger) val).doubleValue());
if (val.isInt() || val.isLong()) {
return val.asLong();
}
if (val instanceof String) {
return charsetFix((String) val);
if (val.isNumber()) {
return val.asDouble();
}
if (val instanceof List) {
if (val.isTextual()) {
return charsetFix(val.asText());
}
if (val.isArray()) {
List<Object> newList = new ArrayList<>();
for (Object entry : ((List) val)) {
for (Iterator<JsonNode> it = val.iterator(); it.hasNext(); ) {
JsonNode entry = it.next();
newList.add(valueConversionFunction(entry));
}
return newList;
}
if (val instanceof Map) {
if (val.isObject()) {
Map<String, Object> newMap = new LinkedHashMap<>();
Map<String, Object> valMap = (Map<String, Object>) val;
for (Map.Entry<String, Object> entry : valMap.entrySet()) {
for (Iterator<Map.Entry<String, JsonNode>> it = val.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
newMap.put(entry.getKey(), valueConversionFunction(entry.getValue()));
}
return newMap;
@ -210,10 +226,10 @@ public class JSONPathParser implements Parser<String, Object>
}
}
private boolean isFlatList(List<Object> list)
private boolean isFlatList(JsonNode list)
{
for (Object obj : list) {
if ((obj instanceof Map) || (obj instanceof List)) {
for (JsonNode obj : list) {
if (obj.isObject() || obj.isArray()) {
return false;
}
}
@ -233,7 +249,12 @@ public class JSONPathParser implements Parser<String, Object>
/**
* A PATH field uses a JsonPath expression to retrieve the field value
*/
PATH;
PATH,
/**
* A JQ field uses a JsonQuery expression to retrieve the field value
*/
JQ;
}
/**

30
java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java
View File

@ -106,6 +106,11 @@ public class JSONPathParserTest
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.ROOT, "INVALID_ROOT", "INVALID_ROOT_EXPR"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "INVALID_PATH", "INVALID_PATH_EXPR"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar1", ".foo.bar1"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-heybarx0", ".hey[0].barx"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-met-array", ".met.a"));
final Parser<String, Object> jsonParser = new JSONPathParser(fields, true, null);
final Map<String, Object> jsonMap = jsonParser.parse(nestedJson);
@ -139,6 +144,11 @@ public class JSONPathParserTest
Assert.assertEquals("asdf", jsonMap.get("heybarx0"));
Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("met-array"));
Assert.assertEquals("aaa", jsonMap.get("jq-nested-foo.bar1"));
Assert.assertEquals("bbb", jsonMap.get("jq-nested-foo.bar2"));
Assert.assertEquals("asdf", jsonMap.get("jq-heybarx0"));
Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("jq-met-array"));
// Fields that should not be discovered
Assert.assertNull(jsonMap.get("hey"));
Assert.assertNull(jsonMap.get("met"));
@ -159,6 +169,9 @@ public class JSONPathParserTest
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "nested-foo.bar2", "$.foo.bar2"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "heybarx0", "$.hey[0].barx"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "met-array", "$.met.a"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-heybarx0", ".hey[0].barx"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-met-array", ".met.a"));
final Parser<String, Object> jsonParser = new JSONPathParser(fields, false, null);
final Map<String, Object> jsonMap = jsonParser.parse(nestedJson);
@ -171,6 +184,9 @@ public class JSONPathParserTest
Assert.assertEquals("bbb", jsonMap.get("nested-foo.bar2"));
Assert.assertEquals("asdf", jsonMap.get("heybarx0"));
Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("met-array"));
Assert.assertEquals("bbb", jsonMap.get("jq-nested-foo.bar2"));
Assert.assertEquals("asdf", jsonMap.get("jq-heybarx0"));
Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("jq-met-array"));
// Fields that should not be discovered
Assert.assertNull(jsonMap.get("newmet"));
@ -198,6 +214,20 @@ public class JSONPathParserTest
final Map<String, Object> jsonMap = jsonParser.parse(nestedJson);
}
@Test
public void testRejectDuplicates2()
{
List<JSONPathParser.FieldSpec> fields = new ArrayList<>();
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "met-array", "$.met.a"));
fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "met-array", ".met.a"));
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("Cannot have duplicate field definition: met-array");
final Parser<String, Object> jsonParser = new JSONPathParser(fields, false, null);
final Map<String, Object> jsonMap = jsonParser.parse(nestedJson);
}
@Test
public void testParseFail()
{

5
pom.xml
View File

@ -679,6 +679,11 @@
<artifactId>json-path</artifactId>
<version>2.1.0</version>
</dependency>
<dependency>
<groupId>net.thisptr</groupId>
<artifactId>jackson-jq</artifactId>
<version>0.0.7</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>