From cbbef80c7f717e9bc554aa58595f86cd68cb537f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 18 Feb 2021 20:56:33 -0800 Subject: [PATCH] add SQL operators for bitwise expressions (#10823) * add SQL operators for bitwise expressions * more test * fix spelling * more tests --- .../apache/druid/math/expr/FunctionTest.java | 20 +++- .../druid/math/expr/VectorExprSanityTest.java | 2 +- docs/querying/sql.md | 8 ++ .../expression/OperatorConversions.java | 40 ++++++++ .../calcite/planner/DruidOperatorTable.java | 24 +++++ .../druid/sql/calcite/CalciteQueryTest.java | 58 +++++++++++ .../calcite/expression/ExpressionsTest.java | 96 +++++++++++++++++++ website/i18n/en.json | 9 ++ 8 files changed, 255 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 96a22840eb0..0d4fb4dbcd1 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -522,6 +522,7 @@ public class FunctionTest extends InitializedNullHandlingTest @Test public void testBitwise() { + // happy path maths assertExpr("bitwiseAnd(3, 1)", 1L); assertExpr("bitwiseAnd(2, 1)", 0L); assertExpr("bitwiseOr(3, 1)", 3L); @@ -531,8 +532,17 @@ public class FunctionTest extends InitializedNullHandlingTest assertExpr("bitwiseShiftLeft(2, 1)", 4L); assertExpr("bitwiseShiftRight(2, 1)", 1L); assertExpr("bitwiseAnd(bitwiseComplement(1), 7)", 6L); + + // funny types + // two strings is sad assertExpr("bitwiseAnd('2', '1')", null); - assertExpr("bitwiseAnd(2, '1')", 0L); + // but one is ok, druid forgives you + assertExpr("bitwiseAnd(3, '1')", 1L); + assertExpr("bitwiseAnd(2, null)", NullHandling.replaceWithDefault() ? 0L : null); + + // unary doesn't accept any slop + assertExpr("bitwiseComplement('1')", null); + assertExpr("bitwiseComplement(null)", null); // doubles are cast assertExpr("bitwiseOr(2.345, 1)", 3L); @@ -552,6 +562,14 @@ public class FunctionTest extends InitializedNullHandlingTest assertExpr("bitwiseConvertDoubleToLongBits(bitwiseConvertDoubleToLongBits(2.0))", 4886405595696988160L); assertExpr("bitwiseConvertLongBitsToDouble(4611686018427387904)", 2.0); assertExpr("bitwiseConvertLongBitsToDouble(bitwiseConvertLongBitsToDouble(4611686018427387904))", 1.0E-323); + + // conversion returns null if nonsense inputs + assertExpr("bitwiseConvertLongBitsToDouble('wat')", null); + assertExpr("bitwiseConvertLongBitsToDouble('1')", null); + assertExpr("bitwiseConvertLongBitsToDouble(null)", null); + assertExpr("bitwiseConvertDoubleToLongBits('wat')", null); + assertExpr("bitwiseConvertDoubleToLongBits('1.0')", null); + assertExpr("bitwiseConvertDoubleToLongBits(null)", null); } private void assertExpr(final String expression, @Nullable final Object expectedResult) diff --git a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index 65ae060eb99..f7d0668a6ce 100644 --- a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -103,7 +103,7 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest public void testUnivariateFunctions() { final String[] functions = new String[]{"parse_long"}; - final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)"}; + final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)", "%s(nonexistent)"}; testFunctions(types, templates, functions); } diff --git a/docs/querying/sql.md b/docs/querying/sql.md index 3e83115477f..45f666cdacc 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -379,6 +379,14 @@ to FLOAT. At runtime, Druid will widen 32-bit floats to 64-bit for most expressi |`ATAN2(y, x)`|Angle theta from the conversion of rectangular coordinates (x, y) to polar * coordinates (r, theta).| |`DEGREES(expr)`|Converts an angle measured in radians to an approximately equivalent angle measured in degrees| |`RADIANS(expr)`|Converts an angle measured in degrees to an approximately equivalent angle measured in radians| +|`BITWISE_AND(expr1, expr2)`|Returns the result of `expr1 & expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| +|`BITWISE_COMPLEMENT(expr)`|Returns the result of `~expr`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| +|`BITWISE_CONVERT_DOUBLE_TO_LONG_BITS(expr)`|Converts the bits of an IEEE 754 floating-point double value to a long. If the input is not a double, it is implicitly cast to a double prior to conversion| +|`BITWISE_CONVERT_LONG_BITS_TO_DOUBLE(expr)`|Converts a long to the IEEE 754 floating-point double specified by the bits stored in the long. If the input is not a long, it is implicitly cast to a long prior to conversion| +|`BITWISE_OR(expr1, expr2)`|Returns the result of `expr1 [PIPE] expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| +|`BITWISE_SHIFT_LEFT(expr1, expr2)`|Returns the result of `expr1 << expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| +|`BITWISE_SHIFT_RIGHT(expr1, expr2)`|Returns the result of `expr1 >> expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| +|`BITWISE_XOR(expr1, expr2)`|Returns the result of `expr1 ^ expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles| ### String functions diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java index 1f716e7578c..c7d0780642a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java @@ -50,6 +50,7 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Static; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.apache.druid.segment.column.RowSignature; @@ -612,4 +613,43 @@ public class OperatorConversions return false; } } + + public static DirectOperatorConversion druidUnaryLongFn(String sqlOperator, String druidFunctionName) + { + return new DirectOperatorConversion( + operatorBuilder(sqlOperator) + .requiredOperands(1) + .operandTypes(SqlTypeFamily.NUMERIC) + .returnTypeNullable(SqlTypeName.BIGINT) + .functionCategory(SqlFunctionCategory.NUMERIC) + .build(), + druidFunctionName + ); + } + + public static DirectOperatorConversion druidBinaryLongFn(String sqlOperator, String druidFunctionName) + { + return new DirectOperatorConversion( + operatorBuilder(sqlOperator) + .requiredOperands(2) + .operandTypes(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC) + .returnTypeNullable(SqlTypeName.BIGINT) + .functionCategory(SqlFunctionCategory.NUMERIC) + .build(), + druidFunctionName + ); + } + + public static DirectOperatorConversion druidUnaryDoubleFn(String sqlOperator, String druidFunctionName) + { + return new DirectOperatorConversion( + operatorBuilder(StringUtils.toUpperCase(sqlOperator)) + .requiredOperands(1) + .operandTypes(SqlTypeFamily.NUMERIC) + .returnTypeNullable(SqlTypeName.DOUBLE) + .functionCategory(SqlFunctionCategory.NUMERIC) + .build(), + druidFunctionName + ); + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java index 9fbc8dffb53..7f5ee4a3c51 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java @@ -45,6 +45,7 @@ import org.apache.druid.sql.calcite.aggregation.builtin.SumZeroSqlAggregator; import org.apache.druid.sql.calcite.expression.AliasedOperatorConversion; import org.apache.druid.sql.calcite.expression.BinaryOperatorConversion; import org.apache.druid.sql.calcite.expression.DirectOperatorConversion; +import org.apache.druid.sql.calcite.expression.OperatorConversions; import org.apache.druid.sql.calcite.expression.SqlOperatorConversion; import org.apache.druid.sql.calcite.expression.UnaryFunctionOperatorConversion; import org.apache.druid.sql.calcite.expression.UnaryPrefixOperatorConversion; @@ -237,6 +238,28 @@ public class DruidOperatorTable implements SqlOperatorTable .add(new IPv4AddressStringifyOperatorConversion()) .build(); + private static final List BITWISE_OPERATOR_CONVERSIONS = + ImmutableList.builder() + .add(OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd")) + .add(OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement")) + .add(OperatorConversions.druidBinaryLongFn("BITWISE_OR", "bitwiseOr")) + .add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_LEFT", "bitwiseShiftLeft")) + .add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_RIGHT", "bitwiseShiftRight")) + .add(OperatorConversions.druidBinaryLongFn("BITWISE_XOR", "bitwiseXor")) + .add( + OperatorConversions.druidUnaryLongFn( + "BITWISE_CONVERT_DOUBLE_TO_LONG_BITS", + "bitwiseConvertDoubleToLongBits" + ) + ) + .add( + OperatorConversions.druidUnaryDoubleFn( + "BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", + "bitwiseConvertLongBitsToDouble" + ) + ) + .build(); + private static final List STANDARD_OPERATOR_CONVERSIONS = ImmutableList.builder() .add(new DirectOperatorConversion(SqlStdOperatorTable.ABS, "abs")) @@ -295,6 +318,7 @@ public class DruidOperatorTable implements SqlOperatorTable .addAll(MULTIVALUE_STRING_OPERATOR_CONVERSIONS) .addAll(REDUCTION_OPERATOR_CONVERSIONS) .addAll(IPV4ADDRESS_OPERATOR_CONVERSIONS) + .addAll(BITWISE_OPERATOR_CONVERSIONS) .build(); // Operators that have no conversion, but are handled in the convertlet table, so they still need to exist. diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 6472b33184f..0dc242a5206 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -773,6 +773,64 @@ public class CalciteQueryTest extends BaseCalciteQueryTest ); } + @Test + public void testBitwiseExpressions() throws Exception + { + List expected; + if (useDefault) { + expected = ImmutableList.of( + new Object[]{0L, 7L, 7L, -8L, 28L, 1L, 4607182418800017408L, 3.5E-323}, + new Object[]{325323L, 325323L, 0L, -325324L, 1301292L, 81330L, 4610334938539176755L, 1.60731E-318}, + new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0}, + new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0}, + new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0}, + new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0} + ); + } else { + expected = ImmutableList.of( + new Object[]{null, null, null, -8L, 28L, 1L, 4607182418800017408L, 3.5E-323}, + new Object[]{325323L, 325323L, 0L, -325324L, 1301292L, 81330L, 4610334938539176755L, 1.60731E-318}, + new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0}, + new Object[]{null, null, null, null, null, null, null, null}, + new Object[]{null, null, null, null, null, null, null, null}, + new Object[]{null, null, null, null, null, null, null, null} + ); + } + testQuery( + "SELECT\n" + + "BITWISE_AND(l1, l2),\n" + + "BITWISE_OR(l1, l2),\n" + + "BITWISE_XOR(l1, l2),\n" + + "BITWISE_COMPLEMENT(l1),\n" + + "BITWISE_SHIFT_LEFT(l1, 2),\n" + + "BITWISE_SHIFT_RIGHT(l1, 2),\n" + + "BITWISE_CONVERT_DOUBLE_TO_LONG_BITS(d1),\n" + + "BITWISE_CONVERT_LONG_BITS_TO_DOUBLE(l1)\n" + + "FROM numfoo", + ImmutableList.of( + Druids.newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .columns("v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7") + .virtualColumns( + expressionVirtualColumn("v0", "bitwiseAnd(\"l1\",\"l2\")", ValueType.LONG), + expressionVirtualColumn("v1", "bitwiseOr(\"l1\",\"l2\")", ValueType.LONG), + expressionVirtualColumn("v2", "bitwiseXor(\"l1\",\"l2\")", ValueType.LONG), + expressionVirtualColumn("v3", "bitwiseComplement(\"l1\")", ValueType.LONG), + expressionVirtualColumn("v4", "bitwiseShiftLeft(\"l1\",2)", ValueType.LONG), + expressionVirtualColumn("v5", "bitwiseShiftRight(\"l1\",2)", ValueType.LONG), + expressionVirtualColumn("v6", "bitwiseConvertDoubleToLongBits(\"d1\")", ValueType.LONG), + expressionVirtualColumn("v7", "bitwiseConvertLongBitsToDouble(\"l1\")", ValueType.DOUBLE) + ) + .context(QUERY_CONTEXT_DEFAULT) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .legacy(false) + .build() + ), + expected + ); + } + @Test public void testExplainSelectConstantExpression() throws Exception diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java index 49ed15994f6..898a64c02bd 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java @@ -1982,4 +1982,100 @@ public class ExpressionsTest extends ExpressionTestBase null ); } + + @Test + public void testOperatorConversionsDruidUnaryLongFn() + { + testHelper.testExpression( + OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("a") + ), + DruidExpression.fromExpression("bitwiseComplement(\"a\")"), + -11L + ); + + testHelper.testExpression( + OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("x") + ), + DruidExpression.fromExpression("bitwiseComplement(\"x\")"), + -3L + ); + + testHelper.testExpression( + OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("s") + ), + DruidExpression.fromExpression("bitwiseComplement(\"s\")"), + null + ); + } + + @Test + public void testOperatorConversionsDruidUnaryDoubleFn() + { + testHelper.testExpression( + OperatorConversions.druidUnaryDoubleFn("BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", "bitwiseConvertLongBitsToDouble").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("a") + ), + DruidExpression.fromExpression("bitwiseConvertLongBitsToDouble(\"a\")"), + 4.9E-323 + ); + + testHelper.testExpression( + OperatorConversions.druidUnaryDoubleFn("BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", "bitwiseConvertLongBitsToDouble").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("x") + ), + DruidExpression.fromExpression("bitwiseConvertLongBitsToDouble(\"x\")"), + 1.0E-323 + ); + + testHelper.testExpression( + OperatorConversions.druidUnaryDoubleFn("BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", "bitwiseConvertLongBitsToDouble").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("s") + ), + DruidExpression.fromExpression("bitwiseConvertLongBitsToDouble(\"s\")"), + null + ); + } + + @Test + public void testOperatorConversionsDruidBinaryLongFn() + { + testHelper.testExpression( + OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("a"), + testHelper.makeInputRef("b") + ), + DruidExpression.fromExpression("bitwiseAnd(\"a\",\"b\")"), + 8L + ); + + testHelper.testExpression( + OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("x"), + testHelper.makeInputRef("y") + ), + DruidExpression.fromExpression("bitwiseAnd(\"x\",\"y\")"), + 2L + ); + + testHelper.testExpression( + OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd").calciteOperator(), + ImmutableList.of( + testHelper.makeInputRef("s"), + testHelper.makeInputRef("s") + ), + DruidExpression.fromExpression("bitwiseAnd(\"s\",\"s\")"), + null + ); + } } diff --git a/website/i18n/en.json b/website/i18n/en.json index 9de85a69ad6..f6738099222 100644 --- a/website/i18n/en.json +++ b/website/i18n/en.json @@ -179,6 +179,9 @@ "development/extensions-core/datasketches-tuple": { "title": "DataSketches Tuple Sketch module" }, + "development/extensions-core/druid-aws-rds": { + "title": "Druid AWS RDS Module" + }, "development/extensions-core/druid-basic-security": { "title": "Basic Security" }, @@ -214,6 +217,9 @@ "title": "Amazon Kinesis ingestion", "sidebar_label": "Amazon Kinesis" }, + "development/extensions-core/druid-kubernetes": { + "title": "Kubernetes" + }, "development/extensions-core/lookups-cached-global": { "title": "Globally Cached Lookups" }, @@ -324,6 +330,9 @@ "operations/dump-segment": { "title": "dump-segment tool" }, + "operations/dynamic-config-provider": { + "title": "Dynamic Config Providers" + }, "operations/export-metadata": { "title": "Export Metadata Tool" },