Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add SQL operators for bitwise expressions (#10823) 

* add SQL operators for bitwise expressions

* more test

* fix spelling

* more tests
This commit is contained in:
Clint Wylie 2021-02-18 20:56:33 -08:00 committed by GitHub
parent 84341737d5
commit cbbef80c7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 255 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
core/src/test/java/org/apache/druid/math/expr/FunctionTest.java
View File

@ -522,6 +522,7 @@ public class FunctionTest extends InitializedNullHandlingTest
@Test
public void testBitwise()
{
// happy path maths
assertExpr("bitwiseAnd(3, 1)", 1L);
assertExpr("bitwiseAnd(2, 1)", 0L);
assertExpr("bitwiseOr(3, 1)", 3L);
@ -531,8 +532,17 @@ public class FunctionTest extends InitializedNullHandlingTest
assertExpr("bitwiseShiftLeft(2, 1)", 4L);
assertExpr("bitwiseShiftRight(2, 1)", 1L);
assertExpr("bitwiseAnd(bitwiseComplement(1), 7)", 6L);
// funny types
// two strings is sad
assertExpr("bitwiseAnd('2', '1')", null);
assertExpr("bitwiseAnd(2, '1')", 0L);
// but one is ok, druid forgives you
assertExpr("bitwiseAnd(3, '1')", 1L);
assertExpr("bitwiseAnd(2, null)", NullHandling.replaceWithDefault() ? 0L : null);
// unary doesn't accept any slop
assertExpr("bitwiseComplement('1')", null);
assertExpr("bitwiseComplement(null)", null);
// doubles are cast
assertExpr("bitwiseOr(2.345, 1)", 3L);
@ -552,6 +562,14 @@ public class FunctionTest extends InitializedNullHandlingTest
assertExpr("bitwiseConvertDoubleToLongBits(bitwiseConvertDoubleToLongBits(2.0))", 4886405595696988160L);
assertExpr("bitwiseConvertLongBitsToDouble(4611686018427387904)", 2.0);
assertExpr("bitwiseConvertLongBitsToDouble(bitwiseConvertLongBitsToDouble(4611686018427387904))", 1.0E-323);
// conversion returns null if nonsense inputs
assertExpr("bitwiseConvertLongBitsToDouble('wat')", null);
assertExpr("bitwiseConvertLongBitsToDouble('1')", null);
assertExpr("bitwiseConvertLongBitsToDouble(null)", null);
assertExpr("bitwiseConvertDoubleToLongBits('wat')", null);
assertExpr("bitwiseConvertDoubleToLongBits('1.0')", null);
assertExpr("bitwiseConvertDoubleToLongBits(null)", null);
}
private void assertExpr(final String expression, @Nullable final Object expectedResult)

2
core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java
View File

@ -103,7 +103,7 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest
public void testUnivariateFunctions()
{
final String[] functions = new String[]{"parse_long"};
final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)"};
final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)", "%s(nonexistent)"};
testFunctions(types, templates, functions);
}

8
docs/querying/sql.md
View File

@ -379,6 +379,14 @@ to FLOAT. At runtime, Druid will widen 32-bit floats to 64-bit for most expressi
|`ATAN2(y, x)`|Angle theta from the conversion of rectangular coordinates (x, y) to polar * coordinates (r, theta).|
|`DEGREES(expr)`|Converts an angle measured in radians to an approximately equivalent angle measured in degrees|
|`RADIANS(expr)`|Converts an angle measured in degrees to an approximately equivalent angle measured in radians|
|`BITWISE_AND(expr1, expr2)`|Returns the result of `expr1 & expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles|
|`BITWISE_COMPLEMENT(expr)`|Returns the result of `~expr`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles|
|`BITWISE_CONVERT_DOUBLE_TO_LONG_BITS(expr)`|Converts the bits of an IEEE 754 floating-point double value to a long. If the input is not a double, it is implicitly cast to a double prior to conversion|
|`BITWISE_CONVERT_LONG_BITS_TO_DOUBLE(expr)`|Converts a long to the IEEE 754 floating-point double specified by the bits stored in the long. If the input is not a long, it is implicitly cast to a long prior to conversion|
|`BITWISE_OR(expr1, expr2)`|Returns the result of `expr1 [PIPE] expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles|
|`BITWISE_SHIFT_LEFT(expr1, expr2)`|Returns the result of `expr1 << expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles|
|`BITWISE_SHIFT_RIGHT(expr1, expr2)`|Returns the result of `expr1 >> expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles|
|`BITWISE_XOR(expr1, expr2)`|Returns the result of `expr1 ^ expr2`. Double values will be implicitly cast to longs, use `BITWISE_CONVERT_DOUBLE_TO_LONG_BITS` to perform bitwise operations directly with doubles|
### String functions

40
sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java
View File

@ -50,6 +50,7 @@ import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.Static;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.apache.druid.segment.column.RowSignature;
@ -612,4 +613,43 @@ public class OperatorConversions
return false;
}
}
public static DirectOperatorConversion druidUnaryLongFn(String sqlOperator, String druidFunctionName)
{
return new DirectOperatorConversion(
operatorBuilder(sqlOperator)
.requiredOperands(1)
.operandTypes(SqlTypeFamily.NUMERIC)
.returnTypeNullable(SqlTypeName.BIGINT)
.functionCategory(SqlFunctionCategory.NUMERIC)
.build(),
druidFunctionName
);
}
public static DirectOperatorConversion druidBinaryLongFn(String sqlOperator, String druidFunctionName)
{
return new DirectOperatorConversion(
operatorBuilder(sqlOperator)
.requiredOperands(2)
.operandTypes(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)
.returnTypeNullable(SqlTypeName.BIGINT)
.functionCategory(SqlFunctionCategory.NUMERIC)
.build(),
druidFunctionName
);
}
public static DirectOperatorConversion druidUnaryDoubleFn(String sqlOperator, String druidFunctionName)
{
return new DirectOperatorConversion(
operatorBuilder(StringUtils.toUpperCase(sqlOperator))
.requiredOperands(1)
.operandTypes(SqlTypeFamily.NUMERIC)
.returnTypeNullable(SqlTypeName.DOUBLE)
.functionCategory(SqlFunctionCategory.NUMERIC)
.build(),
druidFunctionName
);
}
}

24
sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java
View File

@ -45,6 +45,7 @@ import org.apache.druid.sql.calcite.aggregation.builtin.SumZeroSqlAggregator;
import org.apache.druid.sql.calcite.expression.AliasedOperatorConversion;
import org.apache.druid.sql.calcite.expression.BinaryOperatorConversion;
import org.apache.druid.sql.calcite.expression.DirectOperatorConversion;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.expression.UnaryFunctionOperatorConversion;
import org.apache.druid.sql.calcite.expression.UnaryPrefixOperatorConversion;
@ -237,6 +238,28 @@ public class DruidOperatorTable implements SqlOperatorTable
.add(new IPv4AddressStringifyOperatorConversion())
.build();
private static final List<SqlOperatorConversion> BITWISE_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd"))
.add(OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_OR", "bitwiseOr"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_LEFT", "bitwiseShiftLeft"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_SHIFT_RIGHT", "bitwiseShiftRight"))
.add(OperatorConversions.druidBinaryLongFn("BITWISE_XOR", "bitwiseXor"))
.add(
OperatorConversions.druidUnaryLongFn(
"BITWISE_CONVERT_DOUBLE_TO_LONG_BITS",
"bitwiseConvertDoubleToLongBits"
)
)
.add(
OperatorConversions.druidUnaryDoubleFn(
"BITWISE_CONVERT_LONG_BITS_TO_DOUBLE",
"bitwiseConvertLongBitsToDouble"
)
)
.build();
private static final List<SqlOperatorConversion> STANDARD_OPERATOR_CONVERSIONS =
ImmutableList.<SqlOperatorConversion>builder()
.add(new DirectOperatorConversion(SqlStdOperatorTable.ABS, "abs"))
@ -295,6 +318,7 @@ public class DruidOperatorTable implements SqlOperatorTable
.addAll(MULTIVALUE_STRING_OPERATOR_CONVERSIONS)
.addAll(REDUCTION_OPERATOR_CONVERSIONS)
.addAll(IPV4ADDRESS_OPERATOR_CONVERSIONS)
.addAll(BITWISE_OPERATOR_CONVERSIONS)
.build();
// Operators that have no conversion, but are handled in the convertlet table, so they still need to exist.

58
sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
View File

@ -773,6 +773,64 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@Test
public void testBitwiseExpressions() throws Exception
{
List<Object[]> expected;
if (useDefault) {
expected = ImmutableList.of(
new Object[]{0L, 7L, 7L, -8L, 28L, 1L, 4607182418800017408L, 3.5E-323},
new Object[]{325323L, 325323L, 0L, -325324L, 1301292L, 81330L, 4610334938539176755L, 1.60731E-318},
new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0},
new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0},
new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0},
new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0}
);
} else {
expected = ImmutableList.of(
new Object[]{null, null, null, -8L, 28L, 1L, 4607182418800017408L, 3.5E-323},
new Object[]{325323L, 325323L, 0L, -325324L, 1301292L, 81330L, 4610334938539176755L, 1.60731E-318},
new Object[]{0L, 0L, 0L, -1L, 0L, 0L, 0L, 0.0},
new Object[]{null, null, null, null, null, null, null, null},
new Object[]{null, null, null, null, null, null, null, null},
new Object[]{null, null, null, null, null, null, null, null}
);
}
testQuery(
"SELECT\n"
+ "BITWISE_AND(l1, l2),\n"
+ "BITWISE_OR(l1, l2),\n"
+ "BITWISE_XOR(l1, l2),\n"
+ "BITWISE_COMPLEMENT(l1),\n"
+ "BITWISE_SHIFT_LEFT(l1, 2),\n"
+ "BITWISE_SHIFT_RIGHT(l1, 2),\n"
+ "BITWISE_CONVERT_DOUBLE_TO_LONG_BITS(d1),\n"
+ "BITWISE_CONVERT_LONG_BITS_TO_DOUBLE(l1)\n"
+ "FROM numfoo",
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE3)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7")
.virtualColumns(
expressionVirtualColumn("v0", "bitwiseAnd(\"l1\",\"l2\")", ValueType.LONG),
expressionVirtualColumn("v1", "bitwiseOr(\"l1\",\"l2\")", ValueType.LONG),
expressionVirtualColumn("v2", "bitwiseXor(\"l1\",\"l2\")", ValueType.LONG),
expressionVirtualColumn("v3", "bitwiseComplement(\"l1\")", ValueType.LONG),
expressionVirtualColumn("v4", "bitwiseShiftLeft(\"l1\",2)", ValueType.LONG),
expressionVirtualColumn("v5", "bitwiseShiftRight(\"l1\",2)", ValueType.LONG),
expressionVirtualColumn("v6", "bitwiseConvertDoubleToLongBits(\"d1\")", ValueType.LONG),
expressionVirtualColumn("v7", "bitwiseConvertLongBitsToDouble(\"l1\")", ValueType.DOUBLE)
)
.context(QUERY_CONTEXT_DEFAULT)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.legacy(false)
.build()
),
expected
);
}
@Test
public void testExplainSelectConstantExpression() throws Exception

96
sql/src/test/java/org/apache/druid/sql/calcite/expression/ExpressionsTest.java
View File

@ -1982,4 +1982,100 @@ public class ExpressionsTest extends ExpressionTestBase
null
);
}
@Test
public void testOperatorConversionsDruidUnaryLongFn()
{
testHelper.testExpression(
OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("a")
),
DruidExpression.fromExpression("bitwiseComplement(\"a\")"),
-11L
);
testHelper.testExpression(
OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("x")
),
DruidExpression.fromExpression("bitwiseComplement(\"x\")"),
-3L
);
testHelper.testExpression(
OperatorConversions.druidUnaryLongFn("BITWISE_COMPLEMENT", "bitwiseComplement").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("s")
),
DruidExpression.fromExpression("bitwiseComplement(\"s\")"),
null
);
}
@Test
public void testOperatorConversionsDruidUnaryDoubleFn()
{
testHelper.testExpression(
OperatorConversions.druidUnaryDoubleFn("BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", "bitwiseConvertLongBitsToDouble").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("a")
),
DruidExpression.fromExpression("bitwiseConvertLongBitsToDouble(\"a\")"),
4.9E-323
);
testHelper.testExpression(
OperatorConversions.druidUnaryDoubleFn("BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", "bitwiseConvertLongBitsToDouble").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("x")
),
DruidExpression.fromExpression("bitwiseConvertLongBitsToDouble(\"x\")"),
1.0E-323
);
testHelper.testExpression(
OperatorConversions.druidUnaryDoubleFn("BITWISE_CONVERT_LONG_BITS_TO_DOUBLE", "bitwiseConvertLongBitsToDouble").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("s")
),
DruidExpression.fromExpression("bitwiseConvertLongBitsToDouble(\"s\")"),
null
);
}
@Test
public void testOperatorConversionsDruidBinaryLongFn()
{
testHelper.testExpression(
OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("a"),
testHelper.makeInputRef("b")
),
DruidExpression.fromExpression("bitwiseAnd(\"a\",\"b\")"),
8L
);
testHelper.testExpression(
OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("x"),
testHelper.makeInputRef("y")
),
DruidExpression.fromExpression("bitwiseAnd(\"x\",\"y\")"),
2L
);
testHelper.testExpression(
OperatorConversions.druidBinaryLongFn("BITWISE_AND", "bitwiseAnd").calciteOperator(),
ImmutableList.of(
testHelper.makeInputRef("s"),
testHelper.makeInputRef("s")
),
DruidExpression.fromExpression("bitwiseAnd(\"s\",\"s\")"),
null
);
}
}

9
website/i18n/en.json
View File

@ -179,6 +179,9 @@
"development/extensions-core/datasketches-tuple": {
"title": "DataSketches Tuple Sketch module"
},
"development/extensions-core/druid-aws-rds": {
"title": "Druid AWS RDS Module"
},
"development/extensions-core/druid-basic-security": {
"title": "Basic Security"
},
@ -214,6 +217,9 @@
"title": "Amazon Kinesis ingestion",
"sidebar_label": "Amazon Kinesis"
},
"development/extensions-core/druid-kubernetes": {
"title": "Kubernetes"
},
"development/extensions-core/lookups-cached-global": {
"title": "Globally Cached Lookups"
},
@ -324,6 +330,9 @@
"operations/dump-segment": {
"title": "dump-segment tool"
},
"operations/dynamic-config-provider": {
"title": "Dynamic Config Providers"
},
"operations/export-metadata": {
"title": "Export Metadata Tool"
},