NIFI-6546 - Add JsonPath set value support

NIFI-6546 - Addressed review comments (double semi-colon, javadoc, error message, null init)
NIFI-6546 - Refactored test and added log error for JsonPath exception based on review
NIFI-6546 - Refactored tests based on review comments
NIFI-6546 - Removed redundant phone check and added constant for empty path
NIFI-6546 - Added brackets based on review comments

This closes #3646

Signed-off-by: Mike Thomsen <mthomsen@apache.org>
This commit is contained in:
mans2singh 2019-08-11 17:59:33 -04:00 committed by Mike Thomsen
parent 254a84d74d
commit 5df6b0edbb
No known key found for this signature in database
GPG Key ID: 88511C3D4CAD246F
8 changed files with 260 additions and 52 deletions

View File

@ -188,6 +188,7 @@ REPLACE : 'replace';
REPLACE_FIRST : 'replaceFirst';
REPLACE_ALL : 'replaceAll';
IF_ELSE : 'ifElse';
JSON_PATH_SET : 'jsonPathSet';
PAD_LEFT : 'padLeft';
PAD_RIGHT : 'padRight';

View File

@ -78,7 +78,7 @@ zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECOD
oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
PREPEND | APPEND | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | JSON_PATH_DELETE | FROM_RADIX) LPAREN! anyArg RPAREN!) |
(TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL | IF_ELSE) LPAREN! anyArg COMMA! anyArg RPAREN!) |
twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL | IF_ELSE | JSON_PATH_SET) LPAREN! anyArg COMMA! anyArg RPAREN!) |
((SUBSTRING | FORMAT | PAD_LEFT | PAD_RIGHT) LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
fiveArgString : GET_DELIMITED_FIELD LPAREN! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg (COMMA! anyArg)?)?)?)? RPAREN!;

View File

@ -29,8 +29,11 @@ import org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionLe
import org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser;
import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.DateEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.DecimalEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
import org.apache.nifi.attribute.expression.language.evaluation.NumberEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.cast.BooleanCastEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.cast.DateCastEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.cast.DecimalCastEvaluator;
@ -63,6 +66,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.IsEmpt
import org.apache.nifi.attribute.expression.language.evaluation.functions.IsNullEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.JsonPathDeleteEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.JsonPathEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.JsonPathSetEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.LastIndexOfEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.LengthEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.LessThanEvaluator;
@ -177,6 +181,7 @@ import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpre
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JOIN;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JSON_PATH;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JSON_PATH_DELETE;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.JSON_PATH_SET;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LAST_INDEX_OF;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LENGTH;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.LESS_THAN;
@ -945,6 +950,29 @@ public class ExpressionCompiler {
return addToken(new JsonPathDeleteEvaluator(toStringEvaluator(subjectEvaluator),
toStringEvaluator(argEvaluators.get(0), "first argument to jsonPathDelete")), "jsonPathDelete");
}
case JSON_PATH_SET: {
verifyArgCount(argEvaluators, 2, "jsonPathSet");
Evaluator<?> valueEvaluator = null;
Evaluator<?> argValueEvaluator = argEvaluators.get(1);
String location = "second argument to jsonPathSet";
if (argValueEvaluator instanceof StringEvaluator) {
valueEvaluator = toStringEvaluator(argValueEvaluator, location);
} else if (argValueEvaluator instanceof DecimalEvaluator) {
valueEvaluator = toDecimalEvaluator(argValueEvaluator, location);
} else if (argValueEvaluator instanceof NumberEvaluator) {
valueEvaluator = toNumberEvaluator(argValueEvaluator, location);
} else if (argValueEvaluator instanceof WholeNumberEvaluator) {
valueEvaluator = toWholeNumberEvaluator(argValueEvaluator, location);
} else if (argValueEvaluator instanceof BooleanEvaluator) {
valueEvaluator = toBooleanEvaluator(argValueEvaluator, location);
} else {
throw new AttributeExpressionLanguageParsingException("Cannot implicitly convert Data Type " +
argValueEvaluator.getResultType() + (location == null ? "" : " at location [" + location + "]"));
}
return addToken(new JsonPathSetEvaluator(toStringEvaluator(subjectEvaluator),
toStringEvaluator(argEvaluators.get(0), "first argument to jsonPathSet"),
valueEvaluator), "jsonPathSet");
}
case IF_ELSE: {
verifyArgCount(argEvaluators, 2, "ifElse");
return addToken(new IfElseEvaluator(toBooleanEvaluator(subjectEvaluator),

View File

@ -23,12 +23,16 @@ import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResul
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* JsonPathEvaluator provides access to document at the specified JsonPath
*/
public class JsonPathEvaluator extends JsonPathBaseEvaluator {
private static final Logger LOGGER = LoggerFactory.getLogger(JsonPathEvaluator.class);
public JsonPathEvaluator(final Evaluator<String> subject, final Evaluator<String> jsonPathExp) {
super(subject, jsonPathExp);
}
@ -43,7 +47,7 @@ public class JsonPathEvaluator extends JsonPathBaseEvaluator {
try {
result = documentContext.read(compiledJsonPath);
} catch (Exception e) {
// assume the path did not match anything in the document
LOGGER.error("Exception while reading JsonPath " + compiledJsonPath.getPath(), e);
return EMPTY_RESULT;
}

View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.attribute.expression.language.evaluation.functions;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import org.apache.nifi.attribute.expression.language.EvaluationContext;
import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
/**
* JsonPathSetEvaluator allows setting values at the specified existing path
*/
public class JsonPathSetEvaluator extends JsonPathBaseEvaluator {
protected Evaluator<?> valueEvaluator;
public JsonPathSetEvaluator(final Evaluator<String> subject, final Evaluator<String> jsonPathExp, final Evaluator<?> valueEvaluator) {
super(subject, jsonPathExp);
this.valueEvaluator = valueEvaluator;
}
@Override
public QueryResult<String> evaluate(EvaluationContext context) {
DocumentContext documentContext = getDocumentContext(context);
final JsonPath compiledJsonPath = getJsonPath(context);
final Object value = valueEvaluator.evaluate(context).getValue();
String result;
try {
result = documentContext.set(compiledJsonPath, value).jsonString();
} catch (Exception e) {
// assume the path did not match anything in the document
return EMPTY_RESULT;
}
return new StringQueryResult(getResultRepresentation(result, EMPTY_RESULT.getValue()));
}
}

View File

@ -38,6 +38,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
@ -60,6 +61,27 @@ import static org.junit.Assert.fail;
public class TestQuery {
// Address book JsonPath constants
public static final String ADDRESS_BOOK_JSON_PATH_FIRST_NAME = "${json:jsonPath('$.firstName')}";
public static final String ADDRESS_BOOK_JSON_PATH_LAST_NAME = "${json:jsonPath('$.lastName')}";
public static final String ADDRESS_BOOK_JSON_PATH_AGE = "${json:jsonPath('$.age')}";
public static final String ADDRESS_BOOK_JSON_PATH_VOTER = "${json:jsonPath('$.voter')}";
public static final String ADDRESS_BOOK_JSON_PATH_ADDRESS_POSTAL_CODE = "${json:jsonPath('$.address.postalCode')}";
public static final String ADDRESS_BOOK_JSON_PATH_PHONE_NUMBERS_TYPE_HOME_NUMBER = "${json:jsonPath(\"$.phoneNumbers[?(@.type=='home')].number\")}";
public static final String ADDRESS_BOOK_JSON_PATH_PHONE_NUMBERS_TYPE_OFFICE_NUMBER = "${json:jsonPath(\"$.phoneNumbers[?(@.type=='office')].number\")}";
public static final String ADDRESS_BOOK_JSON_PATH_HEIGHT = "${json:jsonPath('$.height')}";
public static final String ADDRESS_BOOK_JSON_PATH_EMPTY = "";
private static final List<String> phoneBookAttributes = Arrays.asList(
ADDRESS_BOOK_JSON_PATH_FIRST_NAME,
ADDRESS_BOOK_JSON_PATH_LAST_NAME,
ADDRESS_BOOK_JSON_PATH_AGE,
ADDRESS_BOOK_JSON_PATH_VOTER,
ADDRESS_BOOK_JSON_PATH_ADDRESS_POSTAL_CODE,
ADDRESS_BOOK_JSON_PATH_PHONE_NUMBERS_TYPE_HOME_NUMBER,
ADDRESS_BOOK_JSON_PATH_PHONE_NUMBERS_TYPE_OFFICE_NUMBER
);
@Test
public void testCompilation() {
assertInvalid("${attr:uuid()}");
@ -298,17 +320,12 @@ public class TestQuery {
verifyEquals("${#{test}:append(' - '):append(#{test})}", attributes, stateValues, parameters,"unit - unit");
}
@Test
public void testJsonPath() throws IOException {
final Map<String, String> attributes = new HashMap<>();
attributes.put("json", getResourceAsString("/json/address-book.json"));
verifyEquals("${json:jsonPath('$.firstName')}", attributes, "John");
verifyEquals("${json:jsonPath('$.address.postalCode')}", attributes, "10021-3100");
verifyEquals("${json:jsonPath(\"$.phoneNumbers[?(@.type=='home')].number\")}", attributes, "212 555-1234");
verifyEquals("${json:jsonPath('$.phoneNumbers')}", attributes,
"[{\"type\":\"home\",\"number\":\"212 555-1234\"},{\"type\":\"office\",\"number\":\"646 555-4567\"}]");
verifyEquals("${json:jsonPath('$.missing-path')}", attributes, "");
Map<String,String> attributes = verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_EMPTY,
"", "${json:jsonPathDelete('$.missingpath')}", "");
verifyEquals("${json:jsonPath('$.missingpath')}", attributes, "");
try {
verifyEquals("${json:jsonPath('$..')}", attributes, "");
Assert.fail("Did not detect bad JSON path expression");
@ -327,67 +344,112 @@ public class TestQuery {
}
}
@Test
public void testJsonPathDeleteFirstNameAttribute() throws IOException {
private void verifyAddressBookAttributes(String originalAddressBook, Map<String,String> attributes, String updatedAttribute, Object updatedValue) {
Map<String, String> originalAttributes = new HashMap<>();
originalAttributes.put("json", originalAddressBook);
phoneBookAttributes.stream()
.filter(currentAttribute -> !currentAttribute.equals(updatedAttribute))
.forEach(currentAttribute -> {
String expected = Query.evaluateExpressions(currentAttribute, originalAttributes, null, null, ParameterLookup.EMPTY);
verifyEquals(currentAttribute, attributes, expected);
}
);
if (! ADDRESS_BOOK_JSON_PATH_EMPTY.equals(updatedAttribute) ) {
verifyEquals(updatedAttribute, attributes, updatedValue);
}
}
private Map<String,String> verifyJsonPathExpressions(String targetAttribute, Object originalValue, String updateExpression, Object updatedValue) throws IOException {
final Map<String, String> attributes = new HashMap<>();
String addressBook = getResourceAsString("/json/address-book.json");
attributes.put("json", addressBook);
verifyEquals("${json:jsonPath('$.firstName')}", attributes, "John");
if ( ! ADDRESS_BOOK_JSON_PATH_EMPTY.equals(targetAttribute) ) {
verifyEquals(targetAttribute, attributes, originalValue);
}
String addressBookAfterDelete = Query.evaluateExpressions("${json:jsonPathDelete('$.firstName')}", attributes, ParameterLookup.EMPTY);
String addressBookAfterDelete = Query.evaluateExpressions(updateExpression, attributes, ParameterLookup.EMPTY);
attributes.clear();
attributes.put("json", addressBookAfterDelete);
verifyEquals("${json:jsonPath('$.lastName')}", attributes, "Smith");
verifyEquals("${json:jsonPath('$.age')}", attributes, "25");
verifyEquals("${json:jsonPath('$.address.postalCode')}", attributes, "10021-3100");
verifyEquals("${json:jsonPath(\"$.phoneNumbers[?(@.type=='home')].number\")}", attributes, "212 555-1234");
verifyEquals("${json:jsonPath('$.phoneNumbers')}", attributes,
"[{\"type\":\"home\",\"number\":\"212 555-1234\"},{\"type\":\"office\",\"number\":\"646 555-4567\"}]");
verifyAddressBookAttributes(addressBook, attributes, targetAttribute, updatedValue);
verifyEquals("${json:jsonPath('$.firstName')}", attributes, "");
return attributes;
}
@Test
public void testJsonPathDeleteFirstNameAttribute() throws IOException {
verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_FIRST_NAME,
"John",
"${json:jsonPathDelete('$.firstName')}",
""
);
}
@Test
public void testJsonPathDeleteMissingPath() throws IOException {
final Map<String, String> attributes = new HashMap<>();
String addressBook = getResourceAsString("/json/address-book.json");
attributes.put("json", addressBook);
String addressBookAfterDelete = Query.evaluateExpressions("${json:jsonPathDelete('$.missing-path')}", attributes, ParameterLookup.EMPTY);
attributes.clear();
attributes.put("json", addressBookAfterDelete);
verifyEquals("${json:jsonPath('$.firstName')}", attributes, "John");
verifyEquals("${json:jsonPath('$.lastName')}", attributes, "Smith");
verifyEquals("${json:jsonPath('$.age')}", attributes, "25");
verifyEquals("${json:jsonPath('$.address')}", attributes,
"{\"streetAddress\":\"21 2nd Street\",\"city\":\"New York\",\"state\":\"NY\",\"postalCode\":\"10021-3100\"}");
verifyEquals("${json:jsonPath('$.phoneNumbers')}", attributes,
"[{\"type\":\"home\",\"number\":\"212 555-1234\"},{\"type\":\"office\",\"number\":\"646 555-4567\"}]");
verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_EMPTY,
"",
"${json:jsonPathDelete('$.missing-path')}",
"");
}
@Test
public void testJsonPathDeleteHomePhoneNumber() throws IOException {
final Map<String, String> attributes = new HashMap<>();
String addressBook = getResourceAsString("/json/address-book.json");
attributes.put("json", addressBook);
verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_PHONE_NUMBERS_TYPE_HOME_NUMBER,
"212 555-1234",
"${json:jsonPathDelete(\"$.phoneNumbers[?(@.type=='home')]\")}",
"[]");
}
verifyEquals("${json:jsonPath(\"$.phoneNumbers[?(@.type=='home')].number\")}", attributes, "212 555-1234");
@Test
public void testJsonPathSetFirstNameAttribute() throws IOException {
verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_FIRST_NAME,
"John",
"${json:jsonPathSet('$.firstName', 'James')}",
"James");
}
String addressBookAfterDelete = Query.evaluateExpressions("${json:jsonPathDelete(\"$.phoneNumbers[?(@.type=='home')]\")}", attributes, ParameterLookup.EMPTY);
@Test
public void testJsonPathSetAgeWholeNumberAttribute() throws IOException {
verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_AGE,
"25",
"${json:jsonPathSet('$.age', '35')}",
"35");
}
attributes.clear();
attributes.put("json", addressBookAfterDelete);
@Test
public void testJsonPathSetVoterBooleanAttribute() throws IOException {
verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_VOTER,
"true",
"${json:jsonPathSet('$.voter', false)}",
"false");
}
verifyEquals("${json:jsonPath('$.firstName')}", attributes, "John");
verifyEquals("${json:jsonPath('$.lastName')}", attributes, "Smith");
verifyEquals("${json:jsonPath('$.age')}", attributes, "25");
verifyEquals("${json:jsonPath('$.address.postalCode')}", attributes, "10021-3100");
verifyEquals("${json:jsonPath(\"$.phoneNumbers[?(@.type=='home')].number\")}", attributes, "[]");
verifyEquals("${json:jsonPath('$.phoneNumbers')}", attributes,
"{\"type\":\"office\",\"number\":\"646 555-4567\"}");
@Test
public void testJsonPathSetHeightNumberAttribute() throws IOException {
verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_HEIGHT,
"6.1",
"${json:jsonPathSet('$.height', 5.9)}",
"5.9");
}
@Test
public void testJsonPathSetMissingPathAttribute() throws IOException {
verifyJsonPathExpressions(
ADDRESS_BOOK_JSON_PATH_EMPTY,
"",
"${json:jsonPathSet('$.missing-path', 5.9)}",
"");
}
@Test

View File

@ -1,5 +1,9 @@
{
"firstName": "John", "lastName": "Smith", "age": 25,
"firstName": "John",
"lastName": "Smith",
"age": 25,
"voter" : true,
"height" : 6.1,
"address" : {
"streetAddress": "21 2nd Street",
"city": "New York",

View File

@ -1698,6 +1698,58 @@ form of the updated JSON.#
An empty subject value or a subject value with an invalid JSON document results in an exception bulletin.
[.function]
=== jsonPathSet
*Description*: [.description]#The `jsonPathSet` function sets the value at the specified JsonPath on a Subject JSON and returns string
form of the updated JSON.#
*Subject Type*: [.subject]#String#
*Arguments*:
- [.argName]#_jsonPath_# : [.argDesc]#the JSON path expression to set value on the Subject.#
- [.argName]#_value_# : [.argDesc]#the value expression to be set on the specified path on Subject.#
*Return Type*: [.returnType]#String#
*Examples*: If the "myJson" attribute is
..........
{
"firstName": "John",
"lastName": "Smith",
"age": 25,
"voter" : true,
"height" : 6.1,
"address" : {
"streetAddress": "21 2nd Street",
"city": "New York",
"state": "NY",
"postalCode": "10021-3100"
},
"phoneNumbers": [
{
"type": "home",
"number": "212 555-1234"
},
{
"type": "office",
"number": "646 555-4567"
}
]
}
..........
.jsonPathSet Examples
|===================================================================
| Expression | Value
| `${myJson:jsonPathSet('$.firstName', 'James')}` | `{"firstName":"James", lastName":"Smith", "age":25, "voter":true, "height":6.1, "address":{"streetAddress":"21 2nd Street", "city":"New York", "state":"NY", "postalCode":"10021-3100"}, "phoneNumbers":[{"type":"home", "number":"212 555-1234"}, {"type":"office", "number":"646 555-4567"}]}`
| `${myJson:jsonPathSet('$.missingpath', 'James')}` | Returns original JSON document
|===================================================================
An empty subject value or a subject value with an invalid JSON document results in an exception bulletin.
[[numbers]]
== Mathematical Operations and Numeric Manipulation