Add jq expression support in flattenSpec (#4171)

* add jq expression in the flattenSpec

* more tests

* add benchmark

* fix style

* use JsonNode for both JSONPath and JQ

* clean up

* more clean up

* add documentation

* fix style

* move jackson-jq version to dependencyManagement section. remove commented code

* oops. revert wrong fix

* throw IllegalArgumentException for JQ syntax error

* remove e.printStackTrace() that is forbidden

* touch

api/pom.xml

@@ -87,7 +87,10 @@
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>jsr305</artifactId>
         </dependency>
-
+        <dependency>
+            <groupId>net.thisptr</groupId>
+            <artifactId>jackson-jq</artifactId>
+        </dependency>
         <!-- Tests -->
         <dependency>
             <groupId>junit</groupId>

api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java

@@ -114,6 +114,9 @@ public class JSONParseSpec extends ParseSpec
         case PATH:
           type = JSONPathParser.FieldType.PATH;
           break;
+        case JQ:
+          type = JSONPathParser.FieldType.JQ;
+          break;
         default:
           throw new IllegalArgumentException("Invalid type for field " + druidSpec.getName());
       }

api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java

@@ -69,6 +69,11 @@ public class JSONPathFieldSpec
     return new JSONPathFieldSpec(JSONPathFieldType.PATH, name, expr);
   }
 
+  public static JSONPathFieldSpec createJqField(String name, String expr)
+  {
+    return new JSONPathFieldSpec(JSONPathFieldType.JQ, name, expr);
+  }
+
   public static JSONPathFieldSpec createRootField(String name)
   {
     return new JSONPathFieldSpec(JSONPathFieldType.ROOT, name, null);

api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java

@@ -26,7 +26,8 @@ import io.druid.java.util.common.StringUtils;
 public enum JSONPathFieldType
 {
   ROOT,
-  PATH;
+  PATH,
+  JQ;
 
   @JsonValue
   @Override

api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java

@@ -41,6 +41,9 @@ public class JSONPathSpecTest
     fields.add(JSONPathFieldSpec.createNestedField("hey0barx", "$.hey[0].barx"));
     fields.add(JSONPathFieldSpec.createRootField("timestamp"));
     fields.add(JSONPathFieldSpec.createRootField("foo.bar1"));
+    fields.add(JSONPathFieldSpec.createJqField("foobar1", ".foo.bar1"));
+    fields.add(JSONPathFieldSpec.createJqField("baz0", ".baz[0]"));
+    fields.add(JSONPathFieldSpec.createJqField("hey0barx", ".hey[0].barx"));
 
     JSONPathSpec flattenSpec = new JSONPathSpec(true, fields);
 
@@ -55,6 +58,9 @@ public class JSONPathSpecTest
     JSONPathFieldSpec hey0barx = serdeFields.get(2);
     JSONPathFieldSpec timestamp = serdeFields.get(3);
     JSONPathFieldSpec foodotbar1 = serdeFields.get(4);
+    JSONPathFieldSpec jqFoobar1 = serdeFields.get(5);
+    JSONPathFieldSpec jqBaz0 = serdeFields.get(6);
+    JSONPathFieldSpec jqHey0barx = serdeFields.get(7);
 
     Assert.assertEquals(JSONPathFieldType.PATH, foobar1.getType());
     Assert.assertEquals("foobar1", foobar1.getName());
@@ -68,6 +74,18 @@ public class JSONPathSpecTest
     Assert.assertEquals("hey0barx", hey0barx.getName());
     Assert.assertEquals("$.hey[0].barx", hey0barx.getExpr());
 
+    Assert.assertEquals(JSONPathFieldType.JQ, jqFoobar1.getType());
+    Assert.assertEquals("foobar1", jqFoobar1.getName());
+    Assert.assertEquals(".foo.bar1", jqFoobar1.getExpr());
+
+    Assert.assertEquals(JSONPathFieldType.JQ, jqBaz0.getType());
+    Assert.assertEquals("baz0", jqBaz0.getName());
+    Assert.assertEquals(".baz[0]", jqBaz0.getExpr());
+
+    Assert.assertEquals(JSONPathFieldType.JQ, jqHey0barx.getType());
+    Assert.assertEquals("hey0barx", jqHey0barx.getName());
+    Assert.assertEquals(".hey[0].barx", jqHey0barx.getExpr());
+
     Assert.assertEquals(JSONPathFieldType.ROOT, timestamp.getType());
     Assert.assertEquals("timestamp", timestamp.getName());
     Assert.assertEquals(null, timestamp.getExpr());

benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java

@@ -45,12 +45,15 @@ public class FlattenJSONBenchmark
 
   List<String> flatInputs;
   List<String> nestedInputs;
+  List<String> jqInputs;
   Parser flatParser;
   Parser nestedParser;
+  Parser jqParser;
   Parser fieldDiscoveryParser;
   Parser forcedPathParser;
   int flatCounter = 0;
   int nestedCounter = 0;
+  int jqCounter = 0;
 
   @Setup
   public void prepare() throws Exception
@@ -64,9 +67,14 @@ public class FlattenJSONBenchmark
     for (int i = 0; i < numEvents; i++) {
       nestedInputs.add(gen.generateNestedEvent());
     }
+    jqInputs = new ArrayList<String>();
+    for (int i = 0; i < numEvents; i++) {
+      jqInputs.add(gen.generateNestedEvent()); // reuse the same event as "nested"
+    }
 
     flatParser = gen.getFlatParser();
     nestedParser = gen.getNestedParser();
+    jqParser = gen.getJqParser();
     fieldDiscoveryParser = gen.getFieldDiscoveryParser();
     forcedPathParser = gen.getForcedPathParser();
   }
@@ -91,6 +99,16 @@ public class FlattenJSONBenchmark
     return parsed;
   }
 
+  @Benchmark
+  @BenchmarkMode(Mode.AverageTime)
+  @OutputTimeUnit(TimeUnit.MICROSECONDS)
+  public Map<String, Object> jqflatten()
+  {
+    Map<String, Object> parsed = jqParser.parse(jqInputs.get(jqCounter));
+    jqCounter = (jqCounter + 1) % numEvents;
+    return parsed;
+  }
+
   @Benchmark
   @BenchmarkMode(Mode.AverageTime)
   @OutputTimeUnit(TimeUnit.MICROSECONDS)

benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java

@@ -164,6 +164,46 @@ public class FlattenJSONBenchmarkUtil
     return spec.makeParser();
   }
 
+  public Parser getJqParser()
+  {
+    List<JSONPathFieldSpec> fields = new ArrayList<>();
+    fields.add(JSONPathFieldSpec.createRootField("ts"));
+
+    fields.add(JSONPathFieldSpec.createRootField("d1"));
+    fields.add(JSONPathFieldSpec.createJqField("e1.d1", ".e1.d1"));
+    fields.add(JSONPathFieldSpec.createJqField("e1.d2", ".e1.d2"));
+    fields.add(JSONPathFieldSpec.createJqField("e2.d3", ".e2.d3"));
+    fields.add(JSONPathFieldSpec.createJqField("e2.d4", ".e2.d4"));
+    fields.add(JSONPathFieldSpec.createJqField("e2.d5", ".e2.d5"));
+    fields.add(JSONPathFieldSpec.createJqField("e2.d6", ".e2.d6"));
+    fields.add(JSONPathFieldSpec.createJqField("e2.ad1[0]", ".e2.ad1[0]"));
+    fields.add(JSONPathFieldSpec.createJqField("e2.ad1[1]", ".e2.ad1[1]"));
+    fields.add(JSONPathFieldSpec.createJqField("e2.ad1[2]", ".e2.ad1[2]"));
+    fields.add(JSONPathFieldSpec.createJqField("ae1[0].d1", ".ae1[0].d1"));
+    fields.add(JSONPathFieldSpec.createJqField("ae1[1].d1", ".ae1[1].d1"));
+    fields.add(JSONPathFieldSpec.createJqField("ae1[2].e1.d2", ".ae1[2].e1.d2"));
+
+    fields.add(JSONPathFieldSpec.createRootField("m3"));
+    fields.add(JSONPathFieldSpec.createJqField("e3.m1", ".e3.m1"));
+    fields.add(JSONPathFieldSpec.createJqField("e3.m2", ".e3.m2"));
+    fields.add(JSONPathFieldSpec.createJqField("e3.m3", ".e3.m3"));
+    fields.add(JSONPathFieldSpec.createJqField("e3.m4", ".e3.m4"));
+    fields.add(JSONPathFieldSpec.createJqField("e3.am1[0]", ".e3.am1[0]"));
+    fields.add(JSONPathFieldSpec.createJqField("e3.am1[1]", ".e3.am1[1]"));
+    fields.add(JSONPathFieldSpec.createJqField("e3.am1[2]", ".e3.am1[2]"));
+    fields.add(JSONPathFieldSpec.createJqField("e3.am1[3]", ".e3.am1[3]"));
+    fields.add(JSONPathFieldSpec.createJqField("e4.e4.m4", ".e4.e4.m4"));
+
+    JSONPathSpec flattenSpec = new JSONPathSpec(true, fields);
+    JSONParseSpec spec = new JSONParseSpec(
+        new TimestampSpec("ts", "iso", null),
+        new DimensionsSpec(null, null, null),
+        flattenSpec,
+        null
+    );
+
+    return spec.makeParser();
+  }
 
   public String generateFlatEvent() throws Exception
   {

benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java

@@ -38,12 +38,15 @@ public class FlattenJSONBenchmarkUtilTest
 
     Parser flatParser = eventGen.getFlatParser();
     Parser nestedParser = eventGen.getNestedParser();
+    Parser jqParser = eventGen.getJqParser();
 
     Map<String, Object> event = flatParser.parse(newEvent);
     Map<String, Object> event2 = nestedParser.parse(newEvent2);
+    Map<String, Object> event3 = jqParser.parse(newEvent2);  // reuse the same event as "nested"
 
     checkEvent1(event);
     checkEvent2(event2);
+    checkEvent2(event3); // make sure JQ parser output matches with JSONPath parser output
   }
 
   public void checkEvent1(Map<String, Object> event)

docs/content/ingestion/flatten-json.md

@@ -17,9 +17,9 @@ Defining the JSON Flatten Spec allows nested JSON fields to be flattened during
 
 | Field | Type | Description | Required |
 |-------|------|-------------|----------|
-| type | String | Type of the field, "root" or "path". | yes |
+| type | String | Type of the field, "root", "path" or "jq". | yes |
 | name | String | This string will be used as the column name when the data has been ingested.  | yes |
-| expr | String | Defines an expression for accessing the field within the JSON object, using [JsonPath](https://github.com/jayway/JsonPath) notation. Only used for type "path", otherwise ignored. | only for type "path" |
+| expr | String | Defines an expression for accessing the field within the JSON object, using [JsonPath](https://github.com/jayway/JsonPath) notation for type "path", and [jackson-jq](https://github.com/eiiches/jackson-jq) for type "jq". This field is only used for type "path" and "jq", otherwise ignored. | only for type "path" or "jq" |
 
 Suppose the event JSON has the following form:
 
@@ -99,6 +99,16 @@ To flatten this JSON, the parseSpec could be defined as follows:
         "type": "path",
         "name": "second-food",
         "expr": "$.thing.food[1]"
+      },
+      {
+        "type": "jq",
+        "name": "first-food-by-jq",
+        "expr": ".thing.food[1]"
+      },
+      {
+        "type": "jq",
+        "name": "hello-total",
+        "expr": ".hello | sum"
       }
     ]
   },
@@ -147,3 +157,4 @@ Note that:
 * If auto field discovery is enabled, any discovered field with the same name as one already defined in the field specs will be skipped and not added twice.
 * The JSON input must be a JSON object at the root, not an array. e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported but [{"invalid": "true"}] and [1,2,3] are not.
 * [http://jsonpath.herokuapp.com/](http://jsonpath.herokuapp.com/) is useful for testing the path expressions.
+* jackson-jq supports subset of [./jq](https://stedolan.github.io/jq/) syntax.  Please refer jackson-jq document.

java-util/pom.xml

@@ -107,6 +107,10 @@
             <scope>test</scope>
             <optional>true</optional>
         </dependency>
+      <dependency>
+        <groupId>net.thisptr</groupId>
+        <artifactId>jackson-jq</artifactId>
+      </dependency>
     </dependencies>
 
     <build>

java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.java.util.common.parsers;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.JsonPath;
+import net.thisptr.jackson.jq.JsonQuery;
+import net.thisptr.jackson.jq.exception.JsonQueryException;
+
+
+public class FlattenExpr
+{
+  private JsonPath jsonPathExpr;
+  private JsonQuery jsonQueryExpr;
+
+
+  FlattenExpr(JsonPath jsonPathExpr)
+  {
+    this.jsonPathExpr = jsonPathExpr;
+  }
+
+  FlattenExpr(JsonQuery jsonQueryExpr)
+  {
+    this.jsonQueryExpr = jsonQueryExpr;
+  }
+
+  public JsonNode readPath(JsonNode document, Configuration jsonConfig)
+  {
+    return this.jsonPathExpr.read(document, jsonConfig);
+  }
+
+  public JsonNode readJq(JsonNode document)
+  {
+    try {
+      return this.jsonQueryExpr.apply(document).get(0);
+    }
+    catch (JsonQueryException e) {
+      // ignore errors.
+    }
+    return null;
+  }
+}

java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java

@@ -19,21 +19,23 @@
 
 package io.druid.java.util.common.parsers;
 
-import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Charsets;
 import com.jayway.jsonpath.Configuration;
 import com.jayway.jsonpath.JsonPath;
 import com.jayway.jsonpath.Option;
-import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
+import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider;
 import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
 import io.druid.java.util.common.Pair;
 import io.druid.java.util.common.StringUtils;
+import net.thisptr.jackson.jq.JsonQuery;
+import net.thisptr.jackson.jq.exception.JsonQueryException;
 
-import java.math.BigInteger;
 import java.nio.charset.CharsetEncoder;
 import java.util.ArrayList;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -43,7 +45,7 @@ import java.util.Map;
  */
 public class JSONPathParser implements Parser<String, Object>
 {
-  private final Map<String, Pair<FieldType, JsonPath>> fieldPathMap;
+  private final Map<String, Pair<FieldType, FlattenExpr>> fieldPathMap;
   private final boolean useFieldDiscovery;
   private final ObjectMapper mapper;
   private final CharsetEncoder enc = Charsets.UTF_8.newEncoder();
@@ -65,7 +67,7 @@ public class JSONPathParser implements Parser<String, Object>
 
     // Avoid using defaultConfiguration, as this depends on json-smart which we are excluding.
     this.jsonPathConfig = Configuration.builder()
-                                       .jsonProvider(new JacksonJsonProvider())
+                                       .jsonProvider(new JacksonJsonNodeJsonProvider())
                                        .mappingProvider(new JacksonMappingProvider())
                                        .options(EnumSet.of(Option.SUPPRESS_EXCEPTIONS))
                                        .build();
@@ -94,27 +96,26 @@ public class JSONPathParser implements Parser<String, Object>
   {
     try {
       Map<String, Object> map = new LinkedHashMap<>();
-      Map<String, Object> document = mapper.readValue(
+      JsonNode document = mapper.readValue(input, JsonNode.class);
-          input,
+
-          new TypeReference<Map<String, Object>>()
+      for (Map.Entry<String, Pair<FieldType, FlattenExpr>> entry : fieldPathMap.entrySet()) {
-          {
-          }
-      );
-      for (Map.Entry<String, Pair<FieldType, JsonPath>> entry : fieldPathMap.entrySet()) {
         String fieldName = entry.getKey();
-        Pair<FieldType, JsonPath> pair = entry.getValue();
+        Pair<FieldType, FlattenExpr> pair = entry.getValue();
-        JsonPath path = pair.rhs;
+        FlattenExpr path = pair.rhs;
-        Object parsedVal;
+        JsonNode parsedVal;
         if (pair.lhs == FieldType.ROOT) {
           parsedVal = document.get(fieldName);
+        } else if (pair.lhs == FieldType.PATH) {
+          parsedVal = path.readPath(document, jsonPathConfig);
+        } else if (pair.lhs == FieldType.JQ) {
+          parsedVal = path.readJq(document);
         } else {
-          parsedVal = path.read(document, jsonPathConfig);
+          throw new ParseException("Unknown FieldType", pair.lhs);
         }
         if (parsedVal == null) {
           continue;
         }
-        parsedVal = valueConversionFunction(parsedVal);
+        map.put(fieldName, valueConversionFunction(parsedVal));
-        map.put(fieldName, parsedVal);
       }
       if (useFieldDiscovery) {
         discoverFields(map, document);
@@ -126,70 +127,85 @@ public class JSONPathParser implements Parser<String, Object>
     }
   }
 
-  private Map<String, Pair<FieldType, JsonPath>> generateFieldPaths(List<FieldSpec> fieldSpecs)
+  private Map<String, Pair<FieldType, FlattenExpr>> generateFieldPaths(List<FieldSpec> fieldSpecs)
   {
-    Map<String, Pair<FieldType, JsonPath>> map = new LinkedHashMap<>();
+    Map<String, Pair<FieldType, FlattenExpr>> map = new LinkedHashMap<>();
     for (FieldSpec fieldSpec : fieldSpecs) {
       String fieldName = fieldSpec.getName();
       if (map.get(fieldName) != null) {
         throw new IllegalArgumentException("Cannot have duplicate field definition: " + fieldName);
       }
-      JsonPath path = fieldSpec.getType() == FieldType.PATH ? JsonPath.compile(fieldSpec.getExpr()) : null;
+      FlattenExpr path = null;
-      Pair<FieldType, JsonPath> pair = new Pair<>(fieldSpec.getType(), path);
+      if (fieldSpec.getType() == FieldType.PATH) {
+        path = new FlattenExpr(JsonPath.compile(fieldSpec.getExpr()));
+      } else if (fieldSpec.getType() == FieldType.JQ) {
+        try {
+          path = new FlattenExpr(JsonQuery.compile(fieldSpec.getExpr()));
+        }
+        catch (JsonQueryException e) {
+          throw new IllegalArgumentException("Unable to compile JQ expression: " + fieldSpec.getExpr());
+        }
+      }
+      Pair<FieldType, FlattenExpr> pair = new Pair<>(fieldSpec.getType(), path);
       map.put(fieldName, pair);
     }
     return map;
   }
 
-  private void discoverFields(Map<String, Object> map, Map<String, Object> document)
+  private void discoverFields(Map<String, Object> map, JsonNode document)
   {
-    for (Map.Entry<String, Object> e : document.entrySet()) {
+    for (Iterator<Map.Entry<String, JsonNode>> it = document.fields(); it.hasNext(); ) {
+      Map.Entry<String, JsonNode> e = it.next();
       String field = e.getKey();
       if (!map.containsKey(field)) {
-        Object val = e.getValue();
+        JsonNode val = e.getValue();
-        if (val == null) {
+        if (val.isNull()) {
           continue;
         }
-        if (val instanceof Map) {
+        if (val.isObject()) {
           continue;
         }
-        if (val instanceof List) {
+        if (val.isArray()) {
-          if (!isFlatList((List) val)) {
+          if (!isFlatList(val)) {
             continue;
           }
         }
-        val = valueConversionFunction(val);
+        map.put(field, valueConversionFunction(val));
-        map.put(field, val);
       }
     }
   }
 
-  private Object valueConversionFunction(Object val)
+  private Object valueConversionFunction(JsonNode val)
   {
-    if (val instanceof Integer) {
+    if (val == null) {
-      return Long.valueOf((Integer) val);
+      return null;
     }
 
-    if (val instanceof BigInteger) {
+    if (val.isInt() || val.isLong()) {
-      return Double.valueOf(((BigInteger) val).doubleValue());
+      return val.asLong();
     }
 
-    if (val instanceof String) {
+    if (val.isNumber()) {
-      return charsetFix((String) val);
+      return val.asDouble();
     }
 
-    if (val instanceof List) {
+    if (val.isTextual()) {
+      return charsetFix(val.asText());
+    }
+
+    if (val.isArray()) {
       List<Object> newList = new ArrayList<>();
-      for (Object entry : ((List) val)) {
+      for (Iterator<JsonNode> it = val.iterator(); it.hasNext(); ) {
+        JsonNode entry = it.next();
         newList.add(valueConversionFunction(entry));
       }
       return newList;
     }
 
-    if (val instanceof Map) {
+    if (val.isObject()) {
       Map<String, Object> newMap = new LinkedHashMap<>();
-      Map<String, Object> valMap = (Map<String, Object>) val;
+      for (Iterator<Map.Entry<String, JsonNode>> it = val.fields(); it.hasNext(); ) {
-      for (Map.Entry<String, Object> entry : valMap.entrySet()) {
+        Map.Entry<String, JsonNode> entry = it.next();
         newMap.put(entry.getKey(), valueConversionFunction(entry.getValue()));
       }
       return newMap;
@@ -210,10 +226,10 @@ public class JSONPathParser implements Parser<String, Object>
     }
   }
 
-  private boolean isFlatList(List<Object> list)
+  private boolean isFlatList(JsonNode list)
   {
-    for (Object obj : list) {
+    for (JsonNode obj : list) {
-      if ((obj instanceof Map) || (obj instanceof List)) {
+      if (obj.isObject() || obj.isArray()) {
         return false;
       }
     }
@@ -233,7 +249,12 @@ public class JSONPathParser implements Parser<String, Object>
     /**
      * A PATH field uses a JsonPath expression to retrieve the field value
      */
-    PATH;
+    PATH,
+
+    /**
+     * A JQ field uses a JsonQuery expression to retrieve the field value
+     */
+    JQ;
   }
 
   /**

java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java

@@ -106,6 +106,11 @@ public class JSONPathParserTest
     fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.ROOT, "INVALID_ROOT", "INVALID_ROOT_EXPR"));
     fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "INVALID_PATH", "INVALID_PATH_EXPR"));
 
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar1", ".foo.bar1"));
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2"));
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-heybarx0", ".hey[0].barx"));
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-met-array", ".met.a"));
+
 
     final Parser<String, Object> jsonParser = new JSONPathParser(fields, true, null);
     final Map<String, Object> jsonMap = jsonParser.parse(nestedJson);
@@ -139,6 +144,11 @@ public class JSONPathParserTest
     Assert.assertEquals("asdf", jsonMap.get("heybarx0"));
     Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("met-array"));
 
+    Assert.assertEquals("aaa", jsonMap.get("jq-nested-foo.bar1"));
+    Assert.assertEquals("bbb", jsonMap.get("jq-nested-foo.bar2"));
+    Assert.assertEquals("asdf", jsonMap.get("jq-heybarx0"));
+    Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("jq-met-array"));
+
     // Fields that should not be discovered
     Assert.assertNull(jsonMap.get("hey"));
     Assert.assertNull(jsonMap.get("met"));
@@ -159,6 +169,9 @@ public class JSONPathParserTest
     fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "nested-foo.bar2", "$.foo.bar2"));
     fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "heybarx0", "$.hey[0].barx"));
     fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "met-array", "$.met.a"));
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2"));
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-heybarx0", ".hey[0].barx"));
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-met-array", ".met.a"));
 
     final Parser<String, Object> jsonParser = new JSONPathParser(fields, false, null);
     final Map<String, Object> jsonMap = jsonParser.parse(nestedJson);
@@ -171,6 +184,9 @@ public class JSONPathParserTest
     Assert.assertEquals("bbb", jsonMap.get("nested-foo.bar2"));
     Assert.assertEquals("asdf", jsonMap.get("heybarx0"));
     Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("met-array"));
+    Assert.assertEquals("bbb", jsonMap.get("jq-nested-foo.bar2"));
+    Assert.assertEquals("asdf", jsonMap.get("jq-heybarx0"));
+    Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("jq-met-array"));
 
     // Fields that should not be discovered
     Assert.assertNull(jsonMap.get("newmet"));
@@ -198,6 +214,20 @@ public class JSONPathParserTest
     final Map<String, Object> jsonMap = jsonParser.parse(nestedJson);
   }
 
+  @Test
+  public void testRejectDuplicates2()
+  {
+    List<JSONPathParser.FieldSpec> fields = new ArrayList<>();
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "met-array", "$.met.a"));
+    fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "met-array", ".met.a"));
+
+    thrown.expect(IllegalArgumentException.class);
+    thrown.expectMessage("Cannot have duplicate field definition: met-array");
+
+    final Parser<String, Object> jsonParser = new JSONPathParser(fields, false, null);
+    final Map<String, Object> jsonMap = jsonParser.parse(nestedJson);
+  }
+
   @Test
   public void testParseFail()
   {

pom.xml

@@ -679,6 +679,11 @@
                 <artifactId>json-path</artifactId>
                 <version>2.1.0</version>
             </dependency>
+            <dependency>
+                <groupId>net.thisptr</groupId>
+                <artifactId>jackson-jq</artifactId>
+                <version>0.0.7</version>
+            </dependency>
             <dependency>
                 <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-api</artifactId>