NIFI-2950 Adding support for whole number hex values and a fromRadix function

NIFI-2950 Fixing typo

This closes #1161

nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionLexer.g

@@ -167,6 +167,7 @@ MINUS : 'minus';
 MULTIPLY : 'multiply';
 DIVIDE : 'divide';
 MATH : 'math';
+FROM_RADIX : 'fromRadix';
 TO_RADIX : 'toRadix';
 OR : 'or';
 AND : 'and';

nifi-commons/nifi-expression-language/src/main/antlr3/org/apache/nifi/attribute/expression/language/antlr/AttributeExpressionParser.g

@@ -75,7 +75,7 @@ tokens {
 // functions that return Strings
 zeroArgString : (TO_UPPER | TO_LOWER | TRIM | TO_STRING | URL_ENCODE | URL_DECODE | BASE64_ENCODE | BASE64_DECODE | ESCAPE_JSON | ESCAPE_XML | ESCAPE_CSV | ESCAPE_HTML3 | ESCAPE_HTML4 | UNESCAPE_JSON | UNESCAPE_XML | UNESCAPE_CSV | UNESCAPE_HTML3 | UNESCAPE_HTML4 ) LPAREN! RPAREN!;
 oneArgString : ((SUBSTRING_BEFORE | SUBSTRING_BEFORE_LAST | SUBSTRING_AFTER | SUBSTRING_AFTER_LAST | REPLACE_NULL | REPLACE_EMPTY |
-				PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH) LPAREN! anyArg RPAREN!) |
+				PREPEND | APPEND | FORMAT | STARTS_WITH | ENDS_WITH | CONTAINS | JOIN | JSON_PATH | FROM_RADIX) LPAREN! anyArg RPAREN!) |
 			   (TO_RADIX LPAREN! anyArg (COMMA! anyArg)? RPAREN!);
 twoArgString : ((REPLACE | REPLACE_FIRST | REPLACE_ALL) LPAREN! anyArg COMMA! anyArg RPAREN!) |
 			   (SUBSTRING LPAREN! anyArg (COMMA! anyArg)? RPAREN!);

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/Query.java

@@ -47,6 +47,7 @@ import org.apache.nifi.attribute.expression.language.evaluation.functions.Equals
 import org.apache.nifi.attribute.expression.language.evaluation.functions.CharSequenceTranslatorEvaluator;
 import org.apache.nifi.attribute.expression.language.evaluation.functions.FindEvaluator;
 import org.apache.nifi.attribute.expression.language.evaluation.functions.FormatEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.functions.FromRadixEvaluator;
 import org.apache.nifi.attribute.expression.language.evaluation.functions.GetDelimitedFieldEvaluator;
 import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanEvaluator;
 import org.apache.nifi.attribute.expression.language.evaluation.functions.GreaterThanOrEqualEvaluator;
@@ -123,6 +124,7 @@ import org.antlr.runtime.CharStream;
 import org.antlr.runtime.CommonTokenStream;
 import org.antlr.runtime.tree.Tree;
 
+import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.FROM_RADIX;
 import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.MATH;
 import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_ATTRIBUTES;
 import static org.apache.nifi.attribute.expression.language.antlr.AttributeExpressionParser.ALL_DELINEATED_VALUES;
@@ -1241,6 +1243,10 @@ public class Query {
                             toWholeNumberEvaluator(argEvaluators.get(0)), toWholeNumberEvaluator(argEvaluators.get(1))), "toRadix");
                 }
             }
+            case FROM_RADIX: {
+                return addToken(new FromRadixEvaluator(toStringEvaluator(subjectEvaluator),
+                        toWholeNumberEvaluator(argEvaluators.get(0))), "fromRadix");
+            }
             case MOD: {
                 return addToken(new ModEvaluator(toNumberEvaluator(subjectEvaluator), toNumberEvaluator(argEvaluators.get(0))), "mod");
             }

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/DecimalCastEvaluator.java

@@ -58,7 +58,13 @@ public class DecimalCastEvaluator extends DecimalEvaluator {
                     case DECIMAL:
                         return new DecimalQueryResult(Double.valueOf(trimmed));
                     case WHOLE_NUMBER:
-                        final Long resultValue = Long.valueOf(trimmed);
+                        Long resultValue;
+                        try {
+                            resultValue = Long.valueOf(trimmed);
+                        } catch (NumberFormatException e){
+                            // Will only occur if trimmed is a hex number
+                            resultValue = Long.decode(trimmed);
+                        }
                         return new DecimalQueryResult(resultValue.doubleValue());
                     case NOT_NUMBER:
                     default:

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/NumberCastEvaluator.java

@@ -64,8 +64,14 @@ public class NumberCastEvaluator extends NumberEvaluator {
                     case DECIMAL:
                         return new NumberQueryResult(Double.valueOf(trimmed));
                     case WHOLE_NUMBER:
-                        final Long resultValue = Long.valueOf(trimmed);
+                        Long resultValue;
-                        return new NumberQueryResult(Long.valueOf(trimmed));
+                        try {
+                            resultValue = Long.valueOf(trimmed);
+                        } catch (NumberFormatException e){
+                            // Will only occur if trimmed is a hex number
+                            resultValue = Long.decode(trimmed);
+                        }
+                        return new NumberQueryResult(resultValue);
                     case NOT_NUMBER:
                     default:
                         return new NumberQueryResult(null);

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/cast/WholeNumberCastEvaluator.java

@@ -56,10 +56,17 @@ public class WholeNumberCastEvaluator extends WholeNumberEvaluator {
                 NumberParsing.ParseResultType parseType = NumberParsing.parse(trimmed);
                 switch (parseType){
                     case DECIMAL:
-                        final Double resultValue = Double.valueOf(trimmed);
+                        final Double doubleResultValue = Double.valueOf(trimmed);
-                        return new WholeNumberQueryResult(resultValue.longValue());
+                        return new WholeNumberQueryResult(doubleResultValue.longValue());
                     case WHOLE_NUMBER:
-                        return new WholeNumberQueryResult(Long.valueOf(trimmed));
+                        Long longResultValue;
+                        try {
+                            longResultValue = Long.valueOf(trimmed);
+                        } catch (NumberFormatException e){
+                            // Will only occur if trimmed is a hex number
+                            longResultValue = Long.decode(trimmed);
+                        }
+                        return new WholeNumberQueryResult(longResultValue);
                     case NOT_NUMBER:
                     default:
                         return new WholeNumberQueryResult(null);

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/functions/FromRadixEvaluator.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.attribute.expression.language.evaluation.functions;
+
+import org.apache.nifi.attribute.expression.language.evaluation.Evaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.QueryResult;
+import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberEvaluator;
+import org.apache.nifi.attribute.expression.language.evaluation.WholeNumberQueryResult;
+
+import java.util.Map;
+
+public class FromRadixEvaluator extends WholeNumberEvaluator {
+
+    private final Evaluator<String> numberEvaluator;
+    private final Evaluator<Long> radixEvaluator;
+
+    public FromRadixEvaluator(final Evaluator<String> subject, final Evaluator<Long> radixEvaluator) {
+        this.numberEvaluator = subject;
+        this.radixEvaluator = radixEvaluator;
+    }
+
+    @Override
+    public QueryResult<Long> evaluate(final Map<String, String> attributes) {
+        final String result = numberEvaluator.evaluate(attributes).getValue();
+        if (result == null) {
+            return new WholeNumberQueryResult(null);
+        }
+
+        final Long radix = radixEvaluator.evaluate(attributes).getValue();
+        if (radix == null) {
+            return new WholeNumberQueryResult(null);
+        }
+
+        long longValue = Long.parseLong(result, radix.intValue());
+
+        return new WholeNumberQueryResult(longValue);
+    }
+
+    @Override
+    public Evaluator<?> getSubjectEvaluator() {
+        return numberEvaluator;
+    }
+
+}

nifi-commons/nifi-expression-language/src/main/java/org/apache/nifi/attribute/expression/language/evaluation/util/NumberParsing.java

@@ -21,7 +21,6 @@ import java.util.regex.Pattern;
 
 public class NumberParsing {
 
-    private static final Pattern NUMBER_PATTERN = Pattern.compile("-?\\d+");
 
     public static enum ParseResultType {
         NOT_NUMBER, WHOLE_NUMBER, DECIMAL;
@@ -69,6 +68,8 @@ public class NumberParsing {
 
     private static final Pattern DOUBLE_PATTERN = Pattern.compile(fpRegex);
 
+    private static final Pattern NUMBER_PATTERN = Pattern.compile("-?((\\d+)|(0[xX]" + HexDigits + "))");
+
     private NumberParsing(){
     }
 

nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java

@@ -1057,11 +1057,14 @@ public class TestQuery {
         verifyEquals("${literal(5):toNumber()}", attributes, 5L);
         verifyEquals("${literal(5):toDecimal()}", attributes, 5D);
 
-        // Unquoted doubles are not due to more complicated parsing
         verifyEquals("${literal(\"5.5\")}", attributes, "5.5");
 
-        verifyEquals("${literal(\"5.5\"):toNumber()}", attributes, 5L);
+        verifyEquals("${literal(5.5):toNumber()}", attributes, 5L);
-        verifyEquals("${literal(\"5.5\"):toDecimal()}", attributes, 5.5D);
+        verifyEquals("${literal(5.5):toDecimal()}", attributes, 5.5D);
+        verifyEquals("${literal('0xF.Fp10'):toDecimal()}", attributes, 0xF.Fp10D);
+
+        verifyEquals("${literal('0xABC'):toNumber()}", attributes, 0xABCL);
+        verifyEquals("${literal('-0xABC'):toNumber()}", attributes, -0xABCL);
     }
 
     @Test
@@ -1175,6 +1178,16 @@ public class TestQuery {
         verifyEquals("${filename:substringAfter('-'):toNumber():toRadix(36, 3):toUpper()}", attributes, "073");
     }
 
+    @Test
+    public void testFromRadix() {
+        final Map<String, String> attributes = new HashMap<>();
+        attributes.put("test1", "ABCDEF");
+        attributes.put("test2", "123");
+
+        verifyEquals("${test1:fromRadix(16)}", attributes, 0xABCDEFL);
+        verifyEquals("${test2:fromRadix(4)}", attributes, 27L);
+    }
+
     @Test
     public void testBase64Encode(){
         final Map<String, String> attributes = new HashMap<>();

nifi-docs/src/main/asciidoc/expression-language-guide.adoc

@@ -213,7 +213,11 @@ The Expression Language is generally able to automatically coerce a value of one
 data type for a function. However, functions do exist to manually coerce a value into a specific data type.
 See the <<type_cast>> section for more information.
 
+Hex values are supported for Number and Decimal types but they must be quoted and prepended with "0x" when being
+interpreted as literals. For example these two expressions are valid (without the quotes or "0x" the expression would fail to run properly):
 
+ - ${literal("0xF"):toNumber()}
+ - ${literal("0xF.Fp10"):toDecimal()}
 
 
 
@@ -1644,6 +1648,34 @@ Divide. This is to preserve backwards compatibility and to not force rounding er
 | `${fileSize:toRadix(2, 16)}` | `0000010000000000`
 |=======================================================================================
 
+[.function]
+=== fromRadix
+
+*Description*: [.description]#Converts the Subject from a specified Radix (or number base) to a base ten whole number. The subject will converted as is, without interpretation, and all characters
+must be valid for the base being converted from. For example converting "0xFF" from hex will not work due to "x" being a invalid hex character.
+
+	If a decimal is passed as the subject, it will first be converted to a whole number and then processed.#
+
+*Subject Type*: [.subject]#String#
+
+*Arguments*:
+
+	- [.argName]#_Subject Base_# : [.argDesc]#A Number between 2 and 36 (inclusive)#
+
+*Return Type*: [.returnType]#Number#
+
+*Examples*: If the "fileSize" attributes has a value of 1234A, then the following Expressions will yield
+	the following results:
+
+
+.toRadix Examples
+|=======================================================================================
+| Expression | Value
+| `${fileSize:fromRadix(11)}` | `17720`
+| `${fileSize:fromRadix(16)}` | `74570`
+| `${fileSize:fromRadix(20)}` | `177290`
+|=======================================================================================
+
 [.function]
 === random