Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-2950 Adding support for whole number hex values and a fromRadix function 

NIFI-2950 Fixing typo

This closes #1161
This commit is contained in:
jpercivall 2016-10-26 16:34:14 -04:00 committed by Oleg Zhurakousky
parent e5eda63705
commit c4be800688
10 changed files with 141 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g
View File

@ -167,6 +167,7 @@ MINUS : 'minus';
MULTIPLY : 'multiply';
DIVIDE : 'divide';
MATH : 'math';
FROM_RADIX : 'fromRadix';
TO_RADIX : 'toRadix';
OR : 'or';
AND : 'and';

2
nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g
View File

@ -75,7 +75,7 @@ tokens {
// functions that return Strings
zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 ) LPAREN! RPAREN!;
oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH) LPAREN! anyArg RPAREN!) |
PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
(TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) |
(SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!);

6
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java
View File

@ -47,6 +47,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.Equals
import org.apache.nifi.attribute.expression.language.evaluation.functions.CharSequenceTranslatorEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.FindEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.FormatEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.FromRadixEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.GetDelimitedFieldEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanOrEqualEvaluator;
@ -123,6 +124,7 @@ import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.tree.Tree;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FROM_RADIX;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATH;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_ATTRIBUTES;
import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_DELINEATED_VALUES;
@ -1241,6 +1243,10 @@ public class Query {
toWholeNumberEvaluator(argEvaluators.get(0)), toWholeNumberEvaluator(argEvaluators.get(1))), "toRadix");
}
}
case FROM_RADIX: {
return addToken(new FromRadixEvaluator(toStringEvaluator(subjectEvaluator),
toWholeNumberEvaluator(argEvaluators.get(0))), "fromRadix");
}
case MOD: {
return addToken(new ModEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "mod");
}

8
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java
View File

@ -58,7 +58,13 @@ public class DecimalCastEvaluator extends DecimalEvaluator {
case DECIMAL:
return new DecimalQueryResult(Double.valueOf(trimmed));
case WHOLE_NUMBER:
final Long resultValue = Long.valueOf(trimmed);
Long resultValue;
try {
resultValue = Long.valueOf(trimmed);
} catch (NumberFormatException e){
// Will only occur if trimmed is a hex number
resultValue = Long.decode(trimmed);
}
return new DecimalQueryResult(resultValue.doubleValue());
case NOT_NUMBER:
default:

10
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java
View File

@ -64,8 +64,14 @@ public class NumberCastEvaluator extends NumberEvaluator {
case DECIMAL:
return new NumberQueryResult(Double.valueOf(trimmed));
case WHOLE_NUMBER:
final Long resultValue = Long.valueOf(trimmed);
return new NumberQueryResult(Long.valueOf(trimmed));
Long resultValue;
try {
resultValue = Long.valueOf(trimmed);
} catch (NumberFormatException e){
// Will only occur if trimmed is a hex number
resultValue = Long.decode(trimmed);
}
return new NumberQueryResult(resultValue);
case NOT_NUMBER:
default:
return new NumberQueryResult(null);

13
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java
View File

@ -56,10 +56,17 @@ public class WholeNumberCastEvaluator extends WholeNumberEvaluator {
NumberParsing.ParseResultType parseType = NumberParsing.parse(trimmed);
switch (parseType){
case DECIMAL:
final Double resultValue = Double.valueOf(trimmed);
return new WholeNumberQueryResult(resultValue.longValue());
final Double doubleResultValue = Double.valueOf(trimmed);
return new WholeNumberQueryResult(doubleResultValue.longValue());
case WHOLE_NUMBER:
return new WholeNumberQueryResult(Long.valueOf(trimmed));
Long longResultValue;
try {
longResultValue = Long.valueOf(trimmed);
} catch (NumberFormatException e){
// Will only occur if trimmed is a hex number
longResultValue = Long.decode(trimmed);
}
return new WholeNumberQueryResult(longResultValue);
case NOT_NUMBER:
default:
return new WholeNumberQueryResult(null);

58
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.attribute.expression.language.evaluation.functions;
import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberQueryResult;
import java.util.Map;
public class FromRadixEvaluator extends WholeNumberEvaluator {
private final Evaluator<String> numberEvaluator;
private final Evaluator<Long> radixEvaluator;
public FromRadixEvaluator(final Evaluator<String> subject, final Evaluator<Long> radixEvaluator) {
this.numberEvaluator = subject;
this.radixEvaluator = radixEvaluator;
}
@Override
public QueryResult<Long> evaluate(final Map<String, String> attributes) {
final String result = numberEvaluator.evaluate(attributes).getValue();
if (result == null) {
return new WholeNumberQueryResult(null);
}
final Long radix = radixEvaluator.evaluate(attributes).getValue();
if (radix == null) {
return new WholeNumberQueryResult(null);
}
long longValue = Long.parseLong(result, radix.intValue());
return new WholeNumberQueryResult(longValue);
}
@Override
public Evaluator<?> getSubjectEvaluator() {
return numberEvaluator;
}
}

3
nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java
View File

@ -21,7 +21,6 @@ import java.util.regex.Pattern;
public class NumberParsing {
private static final Pattern NUMBER_PATTERN = Pattern.compile("-?\\d+");
public static enum ParseResultType {
NOT_NUMBER, WHOLE_NUMBER, DECIMAL;
@ -69,6 +68,8 @@ public class NumberParsing {
private static final Pattern DOUBLE_PATTERN = Pattern.compile(fpRegex);
private static final Pattern NUMBER_PATTERN = Pattern.compile("-?((\\d+)|(0[xX]" + HexDigits + "))");
private NumberParsing(){
}

19
nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
View File

@ -1057,11 +1057,14 @@ public class TestQuery {
verifyEquals("${literal(5):toNumber()}", attributes, 5L);
verifyEquals("${literal(5):toDecimal()}", attributes, 5D);
// Unquoted doubles are not due to more complicated parsing
verifyEquals("${literal(\"5.5\")}", attributes, "5.5");
verifyEquals("${literal(\"5.5\"):toNumber()}", attributes, 5L);
verifyEquals("${literal(\"5.5\"):toDecimal()}", attributes, 5.5D);
verifyEquals("${literal(5.5):toNumber()}", attributes, 5L);
verifyEquals("${literal(5.5):toDecimal()}", attributes, 5.5D);
verifyEquals("${literal('0xF.Fp10'):toDecimal()}", attributes, 0xF.Fp10D);
verifyEquals("${literal('0xABC'):toNumber()}", attributes, 0xABCL);
verifyEquals("${literal('-0xABC'):toNumber()}", attributes, -0xABCL);
}
@Test
@ -1175,6 +1178,16 @@ public class TestQuery {
verifyEquals("${filename:substringAfter('-'):toNumber():toRadix(36, 3):toUpper()}", attributes, "073");
}
@Test
public void testFromRadix() {
final Map<String, String> attributes = new HashMap<>();
attributes.put("test1", "ABCDEF");
attributes.put("test2", "123");
verifyEquals("${test1:fromRadix(16)}", attributes, 0xABCDEFL);
verifyEquals("${test2:fromRadix(4)}", attributes, 27L);
}
@Test
public void testBase64Encode(){
final Map<String, String> attributes = new HashMap<>();

32
nifi-docs/src/main/asciidoc/expression-language-guide.adoc
View File

@ -213,7 +213,11 @@ The Expression Language is generally able to automatically coerce a value of one
data type for a function. However, functions do exist to manually coerce a value into a specific data type.
See the <<type_cast>> section for more information.
Hex values are supported for Number and Decimal types but they must be quoted and prepended with "0x" when being
interpreted as literals. For example these two expressions are valid (without the quotes or "0x" the expression would fail to run properly):
- ${literal("0xF"):toNumber()}
- ${literal("0xF.Fp10"):toDecimal()}
@ -1644,6 +1648,34 @@ Divide. This is to preserve backwards compatibility and to not force rounding er
| `${fileSize:toRadix(2, 16)}` | `0000010000000000`
|=======================================================================================
[.function]
=== fromRadix
*Description*: [.description]#Converts the Subject from a specified Radix (or number base) to a base ten whole number. The subject will converted as is, without interpretation, and all characters
must be valid for the base being converted from. For example converting "0xFF" from hex will not work due to "x" being a invalid hex character.
If a decimal is passed as the subject, it will first be converted to a whole number and then processed.#
*Subject Type*: [.subject]#String#
*Arguments*:
- [.argName]#_Subject Base_# : [.argDesc]#A Number between 2 and 36 (inclusive)#
*Return Type*: [.returnType]#Number#
*Examples*: If the "fileSize" attributes has a value of 1234A, then the following Expressions will yield
the following results:
.toRadix Examples
|=======================================================================================
| Expression | Value
| `${fileSize:fromRadix(11)}` | `17720`
| `${fileSize:fromRadix(16)}` | `74570`
| `${fileSize:fromRadix(20)}` | `177290`
|=======================================================================================
[.function]
=== random