Merge pull request #4031 from jfrazee/NIFI-6791

NIFI-6791 Add UUID3 and UUID5 functions to Expression Language
This commit is contained in:
Mike 2020-03-15 10:27:20 -04:00 committed by GitHub
commit 62c34d7e23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 305 additions and 14 deletions

9
NOTICE
View File

@ -113,6 +113,15 @@ This includes derived works from Dropwizard Metrics available under Apache Softw
nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JvmMetrics.java
nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JmxJvmMetrics.java
This product includes derived works from Apache Commons Id (ASLv2 licensed)
Copyright 2003-2006 The Apache Software Foundation
The derived work is adapted from
http://svn.apache.org/repos/asf/commons/sandbox/id/trunk/src/java/org/apache/commons/id/uuid/UUID.java
and can be found in
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/Uuid3Evaluator.java
and
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/Uuid5Evaluator.java
This includes derived works from Cloudera Schema Registry available under Apache Software License V2 (https://github.com/hortonworks/registry)
Cloudera Schema Registry
Copyright 2016-2019 Cloudera, Inc.

View File

@ -81,7 +81,7 @@ outlined by the documentation.
- Generally, when using ANTLR, the preferred method to parse the input is to use a Tree Walker. However, this is far less intuitive for many
Java developers (including those of us who wrote the Expression Language originally). As a result, we instead use ANTLR to tokenize and parse the
input and then obtain an Abstract Syntax Tree and process this "manually" in Java code. This occurs in the Query class.
- We can add the function into our parsing logic by updating the #buildFunctionEvaluator method of the org.apache.nifi.attribute.expression.language.Query class.
- We can add the function into our parsing logic by updating the #buildFunctionEvaluator method of the org.apache.nifi.attribute.expression.compile.ExpressionCompiler class.
A static import will likely need to be added to the Query class in order to reference the new token. The token can then be added to the existing
'case' statement, which will return a new instance of the Evaluator that was just added.
@ -102,4 +102,3 @@ outlined by the documentation.
will require a Subject. In order to see the function, then, you will need to provide a subject, such as typing "${myVariable:" (without the quotes)
and then press Ctrl+Space. This step is important, as it is quite easy to make a mistake when creating the documentation using a free-form text editor,
and this will ensure that users receive a very consistent and quality experience when using the new function.

View File

@ -93,5 +93,15 @@
<artifactId>commons-text</artifactId>
<version>1.8</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.9</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.14</version>
</dependency>
</dependencies>
</project>

View File

@ -183,6 +183,8 @@ TO_LITERAL : 'literal';
JSON_PATH : 'jsonPath';
JSON_PATH_DELETE : 'jsonPathDelete';
REPEAT : 'repeat';
UUID3 : 'UUID3';
UUID5 : 'UUID5';
// 2 arg functions
SUBSTRING : 'substring';

View File

@ -76,7 +76,7 @@ tokens {
// functions that return Strings
zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 | EVALUATE_EL_STRING) LPAREN! RPAREN!;
oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
PREPEND | APPEND | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | JSON_PATH_DELETE | FROM_RADIX) LPAREN! anyArg RPAREN!) |
PREPEND | APPEND | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | JSON_PATH_DELETE | FROM_RADIX | UUID3 | UUID5) LPAREN! anyArg RPAREN!) |
(TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL | IF_ELSE | JSON_PATH_SET | JSON_PATH_ADD) LPAREN! anyArg COMMA! anyArg RPAREN!) |
((SUBSTRING | FORMAT | PAD_LEFT | PAD_RIGHT | REPEAT) LPAREN! anyArg (COMMA! anyArg)? RPAREN!);

View File

@ -112,6 +112,8 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.TrimEv
import org.apache.nifi.attribute.expression.language.evaluation.functions.UrlDecodeEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.UrlEncodeEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.UuidEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.Uuid3Evaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.Uuid5Evaluator;
import org.apache.nifi.attribute.expression.language.evaluation.literals.BooleanLiteralEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.literals.DecimalLiteralEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.literals.StringLiteralEvaluator;
@ -241,6 +243,8 @@ import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpre
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_DECODE;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.URL_ENCODE;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UUID;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UUID3;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.UUID5;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.WHOLE_NUMBER;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.EVALUATE_EL_STRING;
@ -1001,12 +1005,21 @@ public class ExpressionCompiler {
Evaluator<?> argKeyEvaluator = argEvaluators.get(2);
String keyLocation = "third argument to jsonPathPut";
Evaluator<?> keyEvaluator = getJsonPathUpdateEvaluator(argKeyEvaluator, keyLocation);
return addToken(new JsonPathPutEvaluator(toStringEvaluator(subjectEvaluator),
toStringEvaluator(argEvaluators.get(0), "first argument to jsonPathPut"),
toStringEvaluator(keyEvaluator, keyLocation),
valueEvaluator), "jsonPathPut");
}
case UUID3: {
verifyArgCount(argEvaluators, 1, "UUID3");
return addToken(new Uuid3Evaluator(toStringEvaluator(subjectEvaluator),
toStringEvaluator(argEvaluators.get(0), "first argument to UUID3")), "UUID3");
}
case UUID5: {
verifyArgCount(argEvaluators, 1, "UUID5");
return addToken(new Uuid5Evaluator(toStringEvaluator(subjectEvaluator),
toStringEvaluator(argEvaluators.get(0), "first argument to UUID5")), "UUID5");
}
case IF_ELSE: {
verifyArgCount(argEvaluators, 2, "ifElse");
return addToken(new IfElseEvaluator(toBooleanEvaluator(subjectEvaluator),

View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.attribute.expression.language.evaluation.functions;
import org.apache.nifi.attribute.expression.language.EvaluationContext;
import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
import java.nio.ByteBuffer;
import java.util.UUID;
import org.apache.commons.lang3.ArrayUtils;
public class Uuid3Evaluator extends StringEvaluator {
private final Evaluator<String> subject;
private final Evaluator<String> namespace;
public Uuid3Evaluator(final Evaluator<String> subject, final Evaluator<String> namespace) {
this.subject = subject;
this.namespace = namespace;
}
@Override
public QueryResult<String> evaluate(final EvaluationContext evaluationContext) {
final String subjectValue = subject.evaluate(evaluationContext).getValue();
if (subjectValue == null) {
return new StringQueryResult(null);
}
final String nsValue = namespace.evaluate(evaluationContext).getValue();
final UUID nsUUID = nsValue == null ? new UUID(0, 0) : UUID.fromString(nsValue);
final byte[] nsBytes =
ByteBuffer.wrap(new byte[16])
.putLong(nsUUID.getMostSignificantBits())
.putLong(nsUUID.getLeastSignificantBits())
.array();
final byte[] subjectBytes = subjectValue.getBytes();
final byte[] nameBytes = ArrayUtils.addAll(nsBytes, subjectBytes);
return new StringQueryResult(UUID.nameUUIDFromBytes(nameBytes).toString());
}
@Override
public Evaluator<?> getSubjectEvaluator() {
return subject;
}
}

View File

@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.attribute.expression.language.evaluation.functions;
import org.apache.nifi.attribute.expression.language.EvaluationContext;
import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
import org.apache.nifi.attribute.expression.language.evaluation.StringEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.StringQueryResult;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Objects;
import java.util.UUID;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
// This is based on an unreleased implementation in Apache Commons Id. See http://svn.apache.org/repos/asf/commons/sandbox/id/trunk/src/java/org/apache/commons/id/uuid/UUID.java
public class Uuid5Evaluator extends StringEvaluator {
private final Evaluator<String> subject;
private final Evaluator<String> namespace;
public Uuid5Evaluator(final Evaluator<String> subject, final Evaluator<String> namespace) {
this.subject = subject;
this.namespace = namespace;
}
@Override
public QueryResult<String> evaluate(final EvaluationContext evaluationContext) {
final String subjectValue = subject.evaluate(evaluationContext).getValue();
if (subjectValue == null) {
return new StringQueryResult(null);
}
final String nsValue = namespace.evaluate(evaluationContext).getValue();
final UUID nsUUID = nsValue == null ? new UUID(0, 0) : UUID.fromString(nsValue);
final byte[] nsBytes =
ByteBuffer.wrap(new byte[16])
.putLong(nsUUID.getMostSignificantBits())
.putLong(nsUUID.getLeastSignificantBits())
.array();
final byte[] subjectBytes = subjectValue.getBytes();
final byte[] nameBytes = ArrayUtils.addAll(nsBytes, subjectBytes);
final byte[] nameHash = DigestUtils.sha1(nameBytes);
final byte[] uuidBytes = Arrays.copyOf(nameHash, 16);
uuidBytes[6] &= 0x0F;
uuidBytes[6] |= 0x50;
uuidBytes[8] &= 0x3f;
uuidBytes[8] |= 0x80;
return new StringQueryResult(toString(uuidBytes));
}
@Override
public Evaluator<?> getSubjectEvaluator() {
return subject;
}
private static String toString(final byte[] uuid) {
if (uuid == null) {
return new UUID(0, 0).toString();
}
final String encoded = Hex.encodeHexString(Objects.requireNonNull(uuid));
final StringBuffer sb = new StringBuffer(encoded);
while (sb.length() != 32) {
sb.insert(0, "0");
}
sb.ensureCapacity(32);
sb.insert(8, '-');
sb.insert(13, '-');
sb.insert(18, '-');
sb.insert(23, '-');
return sb.toString();
}
}

View File

@ -89,6 +89,10 @@ public class TestQuery {
assertInvalid("${attr:indexOf(length())}");
assertValid("${UUID()}");
assertInvalid("${UUID():nextInt()}");
assertValid("${attr:UUID3('94c09378-43a6-11ea-8bcc-acde48001122')}");
assertValid("${attr:UUID5('94c09378-43a6-11ea-8bcc-acde48001122')}");
assertInvalid("${UUID3('94c09378-43a6-11ea-8bcc-acde48001122', attr)}");
assertInvalid("${UUID5('94c09378-43a6-11ea-8bcc-acde48001122', attr)}");
assertValid("${nextInt()}");
assertValid("${now():format('yyyy/MM/dd')}");
assertInvalid("${attr:times(3)}");
@ -2137,6 +2141,58 @@ public class TestQuery {
}
}
@Test
public void testUuidsWithNamespace() {
// Testing a lot of cases here b/c it's a custom UUID3/5 implementation
final Map<String, String> attributes = new HashMap<>();
attributes.put("myattr0", "u5IkOYFFvYioYBJSNI2XNjPaVoRjXYnr");
attributes.put("myattr1", "mSDgSKQrY67QCTPatV5qHrZa4oUQ2wEX");
attributes.put("myattr2", "u6jH7pF8iAqwjr42i3r5DubdNcgwqEaX");
attributes.put("myattr3", "9eDG1KbqvHrtIMSmvH44t0K7fHXs7xtz");
attributes.put("myattr4", "QeAUDsMYoHJHLsy1BPPSmQWKhCKvwEpj");
attributes.put("myattr5", "U5Cw4b79SW1YiB5Va3DfUMI9y4iJwnVS");
attributes.put("myattr6", "Ig51Jl3EtwaKlVo9MnDSDdJSlXMgZ1It");
attributes.put("myattr7", "F2iLLLHXgliEpIDwJ4JcqeWBVi70cHS6");
attributes.put("myattr8", "1BFShkKLOcjwn1GMsyO4Fmb0iNTVt2Tf");
attributes.put("myattr9", "WxiyO8Gzw0jQnBlYeZMcdNTwCWJe5MNg");
attributes.put("myattr10", null);
// Version 3s
verifyEquals("${myattr0:UUID3('b9e81de3-7047-4b5e-a822-8fff5b49f808')}", attributes, "7ab88cc4-7748-3214-812a-1bc4500a911a");
verifyEquals("${myattr1:UUID3('341857cc-c5f3-4f76-b336-169f81e9dc7a')}", attributes, "d788c2df-95e1-33aa-a548-9be42f222909");
verifyEquals("${myattr2:UUID3('27e35966-52c9-48ba-bc91-3894a2f164d8')}", attributes, "e960f7af-5eec-3298-8512-e3933836bd5a");
verifyEquals("${myattr3:UUID3('1aef683a-2c0b-4f0e-9287-792361873e8f')}", attributes, "bf1727d8-93d3-3550-9071-78c8686f30c3");
verifyEquals("${myattr4:UUID3('5f15efac-e274-42b1-8d0f-15c2c97acb7d')}", attributes, "9e68a780-090d-30a9-903c-22cf7eb5c511");
verifyEquals("${myattr5:UUID3('ebd71811-fd78-4929-856b-4cec7a38d666')}", attributes, "a2a4b1b5-d93f-3656-be0d-f1db281060c1");
verifyEquals("${myattr6:UUID3('7b1bce89-f12b-4b56-afb8-f9b0a1334926')}", attributes, "8eea2153-d42e-3f63-892c-33ff7f0be389");
verifyEquals("${myattr7:UUID3('fe085c56-95e2-4cf8-8612-ba878ed35f0b')}", attributes, "3cd6470f-5432-3599-82ce-1f2b22adcec6");
verifyEquals("${myattr8:UUID3('2be146a5-f54e-4ca1-a10d-d219e9fc6c6f')}", attributes, "c3ed8ced-b32f-39da-a7ea-75934f419446");
verifyEquals("${myattr9:UUID3('4939d5dd-51c1-4d0e-badd-77fa7c7eebc1')}", attributes, "6507198b-f565-3196-9123-6f946f8c53bc");
verifyEmpty("${myattr10:UUID3('4939d5dd-51c1-4d0e-badd-77fa7c7eebc1')}", attributes);
verifyEmpty("${myattr11:UUID3('4939d5dd-51c1-4d0e-badd-77fa7c7eebc1')}", attributes);
verifyEquals("${myattr9:UUID3(${myattr11})}", attributes, "f2d25da2-cc06-34de-80a3-cf64aff82020");
// Version 5s
verifyEquals("${myattr0:UUID5('245b55a8-397d-4480-a41e-16603c8cf9ad')}", attributes, "74f6dc12-6d84-500c-9583-e9fed79912ea");
verifyEquals("${myattr1:UUID5('45089bfa-f5eb-40e3-bc02-4270ccb8ef34')}", attributes, "7b197702-0ed0-5494-9f61-417e26010308");
verifyEquals("${myattr2:UUID5('49861367-c791-4d6d-987e-fe994b2ee4b7')}", attributes, "7b38b455-a0d6-53bd-a0fa-d0f3bf4e7399");
verifyEquals("${myattr3:UUID5('1142b2d9-e434-4931-b1a5-6dbf363aa9cf')}", attributes, "cd13422e-b030-547b-807b-a868e9282eab");
verifyEquals("${myattr4:UUID5('967190d3-b4ba-4ef3-a8e6-3b8bf2d3f1d8')}", attributes, "e4f1ef89-0d25-55cd-bc4b-1904813c3137");
verifyEquals("${myattr5:UUID5('2942f01d-82df-40ee-b1fd-476542160b7c')}", attributes, "a0415b30-5ef9-5530-93a2-fd20f4262d68");
verifyEquals("${myattr6:UUID5('3a47c04b-7cea-4c95-a379-018e64c701c5')}", attributes, "e1931aad-30e8-5283-8505-394f0d08b181");
verifyEquals("${myattr7:UUID5('6f78ce33-4186-46c0-ae05-15f8c78024cf')}", attributes, "a5c80e26-88d6-5de9-9234-66a050f4d940");
verifyEquals("${myattr8:UUID5('b85962a8-6614-49f4-8fdd-a984cf35144e')}", attributes, "6e33ce29-d3f0-59ee-a864-c05ec4d4300d");
verifyEquals("${myattr9:UUID5('5b6da974-4eca-4c17-bbd2-3b59a1b40bee')}", attributes, "2e2e846c-1cbc-54b2-96f7-5a66d246126f");
verifyEmpty("${myattr10:UUID5('4939d5dd-51c1-4d0e-badd-77fa7c7eebc1')}", attributes);
verifyEmpty("${myattr11:UUID5('4939d5dd-51c1-4d0e-badd-77fa7c7eebc1')}", attributes);
verifyEquals("${myattr9:UUID5(${myattr11})}", attributes, "0231a7bf-7bbe-5a0c-8bbd-9c7bc2e95071");
// Make sure it works using the UUID() expression for the namespace
verifyEquals("${myattr0:UUID3(${UUID()}):length()}", attributes, 36L);
verifyEquals("${myattr0:UUID5(${UUID()}):length()}", attributes, 36L);
}
private void verifyEquals(final String expression, final Map<String, String> attributes, final Object expectedResult) {
verifyEquals(expression,attributes, null, ParameterLookup.EMPTY, expectedResult);
}

View File

@ -1426,6 +1426,44 @@ Each of the following functions will encode a string according the rules of the
[.function]
=== UUID3
*Description*: [.description]#Returns a type 3 (MD5 hashed) namespace name-based UUID.#
*Subject Type*: [.subject]#String#
*Arguments*:
- [.argName]#_namespace_# : [.argDesc]#The namespace UUID identifier#
*Return Type*: [.returnType]#String#
*Examples*: ${attr:UUID3('b9e81de3-7047-4b5e-a822-8fff5b49f808')} returns a value similar to 7ab88cc4-7748-3214-812a-1bc4500a911a
[.function]
=== UUID5
*Description*: [.description]#Returns a type 5 (SHA-1 hashed) namespace name-based UUID.#
*Subject Type*: [.subject]#String#
*Arguments*:
- [.argName]#_namespace_# : [.argDesc]#The namespace UUID identifier#
*Return Type*: [.returnType]#String#
*Examples*: ${attr:UUID5('245b55a8-397d-4480-a41e-16603c8cf9ad')} returns a value similar to 6448f0c0-fd2b-30ad-b05a-deb456f8b060
[[searching]]
== Searching
@ -2330,7 +2368,7 @@ an error when validating the function.
[.function]
=== UUID
*Description*: [.description]#Returns a randomly generated UUID.#
*Description*: [.description]#Returns a randomly generated type 4 UUID.#
*Subject Type*: [.subjectless]#No Subject#