Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SQL: Full TRIM support. (#4750) 

* SQL: Full TRIM support.

- Support trimming arbitrary characters
- Support BOTH, LEADING, and TRAILING

* Remove unused import.

* Fix tests, add RTRIM / LTRIM.

* Remove unused imports.

* BTRIM and docs.

* Replace for with foreach.
This commit is contained in:
Gian Merlino 2017-09-12 11:49:08 -07:00 committed by Fangjin Yang
parent b5e839b3db
commit 4909c48b0c
23 changed files with 838 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
common/src/main/java/io/druid/math/expr/Function.java
View File

@ -990,26 +990,6 @@ interface Function
}
}
class TrimFunc implements Function
{
@Override
public String name()
{
return "trim";
}
@Override
public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
{
if (args.size() != 1) {
throw new IAE("Function[%s] needs 1 argument", name());
}
final String arg = args.get(0).eval(bindings).asString();
return ExprEval.of(Strings.nullToEmpty(arg).trim());
}
}
class LowerFunc implements Function
{
@Override

6
common/src/test/java/io/druid/math/expr/FunctionTest.java
View File

@ -85,12 +85,6 @@ public class FunctionTest
assertExpr("strlen(nonexistent)", 0L);
}
@Test
public void testTrim()
{
assertExpr("trim(concat(' ',x,' '))", "foo");
}
@Test
public void testLower()
{

10
docs/content/misc/math-expr.md
View File

@ -42,10 +42,12 @@ Also, the following built-in functions are supported.
|regexp_extract|regexp_extract(expr, pattern[, index]) applies a regular expression pattern and extracts a capture group index, or null if there is no match. If index is unspecified or zero, returns the substring that matched the pattern.|
|replace|replace(expr, pattern, replacement) replaces pattern with replacement|
|substring|substring(expr, index, length) behaves like java.lang.String's substring|
|strlen|returns length of a string in UTF-16 code units|
|trim|remove leading and trailing whitespace from a string|
|lower|convert a string to lowercase|
|upper|convert a string to uppercase|
|strlen|strlen(expr) returns length of a string in UTF-16 code units|
|trim|trim(expr[, chars]) remove leading and trailing characters from `expr` if they are present in `chars`. `chars` defaults to ' ' (space) if not provided.|
|ltrim|ltrim(expr[, chars]) remove leading characters from `expr` if they are present in `chars`. `chars` defaults to ' ' (space) if not provided.|
|rtrim|rtrim(expr[, chars]) remove trailing characters from `expr` if they are present in `chars`. `chars` defaults to ' ' (space) if not provided.|
|lower|lower(expr) converts a string to lowercase|
|upper|upper(expr) converts a string to uppercase|
## Time functions

5
docs/content/querying/sql.md
View File

@ -126,7 +126,10 @@ String functions accept strings, and return a type appropriate to the function.
|`REGEXP_EXTRACT(expr, pattern, [index])`|Apply regular expression pattern and extract a capture group, or null if there is no match. If index is unspecified or zero, returns the substring that matched the pattern.|
|`REPLACE(expr, pattern, replacement)`|Replaces pattern with replacement in expr, and returns the result.|
|`SUBSTRING(expr, index, [length])`|Returns a substring of expr starting at index, with a max length, both measured in UTF-16 code units.|
|`TRIM(expr)`|Returns expr with leading and trailing whitespace removed.|
|`TRIM([BOTH | LEADING | TRAILING] [<chars> FROM] expr)`|Returns expr with characters removed from the leading, trailing, or both ends of "expr" if they are in "chars". If "chars" is not provided, it defaults to " " (a space). If the directional argument is not provided, it defaults to "BOTH".|
|`BTRIM(expr[, chars])`|Alternate form of `TRIM(BOTH <chars> FROM <expr>`).|
|`LTRIM(expr[, chars])`|Alternate form of `TRIM(LEADING <chars> FROM <expr>`).|
|`RTRIM(expr[, chars])`|Alternate form of `TRIM(TRAILING <chars> FROM <expr>`).|
|`UPPER(expr)`|Returns expr in all uppercase.|
### Time functions

276
processing/src/main/java/io/druid/query/expression/TrimExprMacro.java Normal file
View File

@ -0,0 +1,276 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.query.expression;
import io.druid.java.util.common.IAE;
import io.druid.math.expr.Expr;
import io.druid.math.expr.ExprEval;
import io.druid.math.expr.ExprMacroTable;
import javax.annotation.Nonnull;
import java.util.List;
public abstract class TrimExprMacro implements ExprMacroTable.ExprMacro
{
private static final char[] EMPTY_CHARS = new char[0];
private static final char[] DEFAULT_CHARS = new char[]{' '};
enum TrimMode
{
BOTH(true, true),
LEFT(true, false),
RIGHT(false, true);
private final boolean left;
private final boolean right;
TrimMode(final boolean left, final boolean right)
{
this.left = left;
this.right = right;
}
public boolean isLeft()
{
return left;
}
public boolean isRight()
{
return right;
}
}
private final TrimMode mode;
private final String name;
public TrimExprMacro(final String name, final TrimMode mode)
{
this.name = name;
this.mode = mode;
}
@Override
public String name()
{
return name;
}
@Override
public Expr apply(final List<Expr> args)
{
if (args.size() < 1 || args.size() > 2) {
throw new IAE("Function[%s] must have 1 or 2 arguments", name());
}
if (args.size() == 1) {
return new TrimStaticCharsExpr(mode, args.get(0), DEFAULT_CHARS);
} else {
final Expr charsArg = args.get(1);
if (charsArg.isLiteral()) {
final String charsString = charsArg.eval(ExprUtils.nilBindings()).asString();
final char[] chars = charsString == null ? EMPTY_CHARS : charsString.toCharArray();
return new TrimStaticCharsExpr(mode, args.get(0), chars);
} else {
return new TrimDynamicCharsExpr(mode, args.get(0), args.get(1));
}
}
}
private static class TrimStaticCharsExpr implements Expr
{
private final TrimMode mode;
private final Expr stringExpr;
private final char[] chars;
public TrimStaticCharsExpr(final TrimMode mode, final Expr stringExpr, final char[] chars)
{
this.mode = mode;
this.stringExpr = stringExpr;
this.chars = chars;
}
@Nonnull
@Override
public ExprEval eval(final ObjectBinding bindings)
{
final ExprEval stringEval = stringExpr.eval(bindings);
if (chars.length == 0 || stringEval.isNull()) {
return stringEval;
}
final String s = stringEval.asString();
int start = 0;
int end = s.length();
if (mode.isLeft()) {
while (start < s.length()) {
if (arrayContains(chars, s.charAt(start))) {
start++;
} else {
break;
}
}
}
if (mode.isRight()) {
while (end > start) {
if (arrayContains(chars, s.charAt(end - 1))) {
end--;
} else {
break;
}
}
}
if (start == 0 && end == s.length()) {
return stringEval;
} else {
return ExprEval.of(s.substring(start, end));
}
}
@Override
public void visit(final Visitor visitor)
{
stringExpr.visit(visitor);
visitor.visit(this);
}
}
private static class TrimDynamicCharsExpr implements Expr
{
private final TrimMode mode;
private final Expr stringExpr;
private final Expr charsExpr;
public TrimDynamicCharsExpr(final TrimMode mode, final Expr stringExpr, final Expr charsExpr)
{
this.mode = mode;
this.stringExpr = stringExpr;
this.charsExpr = charsExpr;
}
@Nonnull
@Override
public ExprEval eval(final ObjectBinding bindings)
{
final ExprEval stringEval = stringExpr.eval(bindings);
if (stringEval.isNull()) {
return stringEval;
}
final ExprEval charsEval = charsExpr.eval(bindings);
if (charsEval.isNull()) {
return stringEval;
}
final String s = stringEval.asString();
final String chars = charsEval.asString();
int start = 0;
int end = s.length();
if (mode.isLeft()) {
while (start < s.length()) {
if (stringContains(chars, s.charAt(start))) {
start++;
} else {
break;
}
}
}
if (mode.isRight()) {
while (end > start) {
if (stringContains(chars, s.charAt(end - 1))) {
end--;
} else {
break;
}
}
}
if (start == 0 && end == s.length()) {
return stringEval;
} else {
return ExprEval.of(s.substring(start, end));
}
}
@Override
public void visit(final Visitor visitor)
{
stringExpr.visit(visitor);
charsExpr.visit(visitor);
visitor.visit(this);
}
}
private static boolean arrayContains(char[] array, char c)
{
for (final char arrayChar : array) {
if (arrayChar == c) {
return true;
}
}
return false;
}
private static boolean stringContains(String string, char c)
{
for (int i = 0; i < string.length(); i++) {
if (string.charAt(i) == c) {
return true;
}
}
return false;
}
public static class BothTrimExprMacro extends TrimExprMacro
{
public BothTrimExprMacro()
{
super("trim", TrimMode.BOTH);
}
}
public static class LeftTrimExprMacro extends TrimExprMacro
{
public LeftTrimExprMacro()
{
super("ltrim", TrimMode.LEFT);
}
}
public static class RightTrimExprMacro extends TrimExprMacro
{
public RightTrimExprMacro()
{
super("rtrim", TrimMode.RIGHT);
}
}
}

37
processing/src/test/java/io/druid/query/expression/ExprMacroTest.java
View File

@ -39,6 +39,7 @@ public class ExprMacroTest
.put("y", 2)
.put("z", 3.1)
.put("CityOfAngels", "America/Los_Angeles")
.put("spacey", " hey there ")
.build()
);
@ -135,6 +136,42 @@ public class ExprMacroTest
assertExpr("timestamp_format(t,'yyyy-MM-dd HH:mm:ss','America/Los_Angeles')", "2000-02-02 20:05:06");
}
@Test
public void testTrim()
{
assertExpr("trim('')", null);
assertExpr("trim(concat(' ',x,' '))", "foo");
assertExpr("trim(spacey)", "hey there");
assertExpr("trim(spacey, '')", " hey there ");
assertExpr("trim(spacey, 'he ')", "y ther");
assertExpr("trim(spacey, spacey)", null);
assertExpr("trim(spacey, substring(spacey, 0, 4))", "y ther");
}
@Test
public void testLTrim()
{
assertExpr("ltrim('')", null);
assertExpr("ltrim(concat(' ',x,' '))", "foo ");
assertExpr("ltrim(spacey)", "hey there ");
assertExpr("ltrim(spacey, '')", " hey there ");
assertExpr("ltrim(spacey, 'he ')", "y there ");
assertExpr("ltrim(spacey, spacey)", null);
assertExpr("ltrim(spacey, substring(spacey, 0, 4))", "y there ");
}
@Test
public void testRTrim()
{
assertExpr("rtrim('')", null);
assertExpr("rtrim(concat(' ',x,' '))", " foo");
assertExpr("rtrim(spacey)", " hey there");
assertExpr("rtrim(spacey, '')", " hey there ");
assertExpr("rtrim(spacey, 'he ')", " hey ther");
assertExpr("rtrim(spacey, spacey)", null);
assertExpr("rtrim(spacey, substring(spacey, 0, 4))", " hey ther");
}
private void assertExpr(final String expression, final Object expectedResult)
{
final Expr expr = Parser.parse(expression, TestExprMacroTable.INSTANCE);

5
processing/src/test/java/io/druid/query/expression/TestExprMacroTable.java
View File

@ -48,7 +48,10 @@ public class TestExprMacroTable extends ExprMacroTable
new TimestampFloorExprMacro(),
new TimestampFormatExprMacro(),
new TimestampParseExprMacro(),
new TimestampShiftExprMacro()
new TimestampShiftExprMacro(),
new TrimExprMacro.BothTrimExprMacro(),
new TrimExprMacro.LeftTrimExprMacro(),
new TrimExprMacro.RightTrimExprMacro()
)
);
}

4
server/src/main/java/io/druid/guice/ExpressionModule.java
View File

@ -34,6 +34,7 @@ import io.druid.query.expression.TimestampFloorExprMacro;
import io.druid.query.expression.TimestampFormatExprMacro;
import io.druid.query.expression.TimestampParseExprMacro;
import io.druid.query.expression.TimestampShiftExprMacro;
import io.druid.query.expression.TrimExprMacro;
import java.util.List;
@ -51,6 +52,9 @@ public class ExpressionModule implements DruidModule
.add(TimestampFormatExprMacro.class)
.add(TimestampParseExprMacro.class)
.add(TimestampShiftExprMacro.class)
.add(TrimExprMacro.BothTrimExprMacro.class)
.add(TrimExprMacro.LeftTrimExprMacro.class)
.add(TrimExprMacro.RightTrimExprMacro.class)
.build();
@Override

76
sql/src/main/java/io/druid/sql/calcite/expression/BTrimOperatorConversion.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.sql.calcite.expression;
import io.druid.sql.calcite.planner.PlannerContext;
import io.druid.sql.calcite.table.RowSignature;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlTrimFunction;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
public class BTrimOperatorConversion implements SqlOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("BTRIM")
.operandTypes(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)
.returnType(SqlTypeName.VARCHAR)
.functionCategory(SqlFunctionCategory.STRING)
.requiredOperands(1)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
@Override
public DruidExpression toDruidExpression(
final PlannerContext plannerContext,
final RowSignature rowSignature,
final RexNode rexNode
)
{
return OperatorConversions.convertCall(
plannerContext,
rowSignature,
rexNode,
druidExpressions -> {
if (druidExpressions.size() > 1) {
return TrimOperatorConversion.makeTrimExpression(
SqlTrimFunction.Flag.BOTH,
druidExpressions.get(0),
druidExpressions.get(1)
);
} else {
return TrimOperatorConversion.makeTrimExpression(
SqlTrimFunction.Flag.BOTH,
druidExpressions.get(0),
DruidExpression.fromExpression(DruidExpression.stringLiteral(" "))
);
}
}
);
}
}

1
sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java
View File

@ -89,7 +89,6 @@ public class Expressions
.put(SqlStdOperatorTable.POWER, "pow")
.put(SqlStdOperatorTable.REPLACE, "replace")
.put(SqlStdOperatorTable.SQRT, "sqrt")
.put(SqlStdOperatorTable.TRIM, "trim")
.put(SqlStdOperatorTable.UPPER, "upper")
.build();

76
sql/src/main/java/io/druid/sql/calcite/expression/LTrimOperatorConversion.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.sql.calcite.expression;
import io.druid.sql.calcite.planner.PlannerContext;
import io.druid.sql.calcite.table.RowSignature;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlTrimFunction;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
public class LTrimOperatorConversion implements SqlOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("LTRIM")
.operandTypes(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)
.returnType(SqlTypeName.VARCHAR)
.functionCategory(SqlFunctionCategory.STRING)
.requiredOperands(1)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
@Override
public DruidExpression toDruidExpression(
final PlannerContext plannerContext,
final RowSignature rowSignature,
final RexNode rexNode
)
{
return OperatorConversions.convertCall(
plannerContext,
rowSignature,
rexNode,
druidExpressions -> {
if (druidExpressions.size() > 1) {
return TrimOperatorConversion.makeTrimExpression(
SqlTrimFunction.Flag.LEADING,
druidExpressions.get(0),
druidExpressions.get(1)
);
} else {
return TrimOperatorConversion.makeTrimExpression(
SqlTrimFunction.Flag.LEADING,
druidExpressions.get(0),
DruidExpression.fromExpression(DruidExpression.stringLiteral(" "))
);
}
}
);
}
}

2
sql/src/main/java/io/druid/sql/calcite/expression/LookupOperatorConversion.java
View File

@ -62,7 +62,7 @@ public class LookupOperatorConversion implements SqlOperatorConversion
final RexNode rexNode
)
{
return OperatorConversions.functionCall(
return OperatorConversions.convertCall(
plannerContext,
rowSignature,
rexNode,

38
sql/src/main/java/io/druid/sql/calcite/expression/OperatorConversions.java
View File

@ -33,6 +33,7 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
@ -42,23 +43,49 @@ import java.util.function.Function;
*/
public class OperatorConversions
{
public static DruidExpression functionCall(
@Nullable
public static DruidExpression convertCall(
final PlannerContext plannerContext,
final RowSignature rowSignature,
final RexNode rexNode,
final String functionName
)
{
return functionCall(plannerContext, rowSignature, rexNode, functionName, null);
return convertCall(
plannerContext,
rowSignature,
rexNode,
druidExpressions -> DruidExpression.fromFunctionCall(functionName, druidExpressions)
);
}
public static DruidExpression functionCall(
@Nullable
public static DruidExpression convertCall(
final PlannerContext plannerContext,
final RowSignature rowSignature,
final RexNode rexNode,
final String functionName,
final Function<List<DruidExpression>, SimpleExtraction> simpleExtractionFunction
)
{
return convertCall(
plannerContext,
rowSignature,
rexNode,
druidExpressions -> DruidExpression.of(
simpleExtractionFunction == null ? null : simpleExtractionFunction.apply(druidExpressions),
DruidExpression.functionCall(functionName, druidExpressions)
)
);
}
@Nullable
public static DruidExpression convertCall(
final PlannerContext plannerContext,
final RowSignature rowSignature,
final RexNode rexNode,
final Function<List<DruidExpression>, DruidExpression> expressionFunction
)
{
final RexCall call = (RexCall) rexNode;
@ -72,10 +99,7 @@ public class OperatorConversions
return null;
}
return DruidExpression.of(
simpleExtractionFunction == null ? null : simpleExtractionFunction.apply(druidExpressions),
DruidExpression.functionCall(functionName, druidExpressions)
);
return expressionFunction.apply(druidExpressions);
}
public static OperatorBuilder operatorBuilder(final String name)

76
sql/src/main/java/io/druid/sql/calcite/expression/RTrimOperatorConversion.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.sql.calcite.expression;
import io.druid.sql.calcite.planner.PlannerContext;
import io.druid.sql.calcite.table.RowSignature;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlTrimFunction;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
public class RTrimOperatorConversion implements SqlOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder("RTRIM")
.operandTypes(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)
.returnType(SqlTypeName.VARCHAR)
.functionCategory(SqlFunctionCategory.STRING)
.requiredOperands(1)
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
@Override
public DruidExpression toDruidExpression(
final PlannerContext plannerContext,
final RowSignature rowSignature,
final RexNode rexNode
)
{
return OperatorConversions.convertCall(
plannerContext,
rowSignature,
rexNode,
druidExpressions -> {
if (druidExpressions.size() > 1) {
return TrimOperatorConversion.makeTrimExpression(
SqlTrimFunction.Flag.TRAILING,
druidExpressions.get(0),
druidExpressions.get(1)
);
} else {
return TrimOperatorConversion.makeTrimExpression(
SqlTrimFunction.Flag.TRAILING,
druidExpressions.get(0),
DruidExpression.fromExpression(DruidExpression.stringLiteral(" "))
);
}
}
);
}
}

2
sql/src/main/java/io/druid/sql/calcite/expression/RegexpExtractOperatorConversion.java
View File

@ -55,7 +55,7 @@ public class RegexpExtractOperatorConversion implements SqlOperatorConversion
final RexNode rexNode
)
{
return OperatorConversions.functionCall(
return OperatorConversions.convertCall(
plannerContext,
rowSignature,
rexNode,

2
sql/src/main/java/io/druid/sql/calcite/expression/TimeParseOperatorConversion.java
View File

@ -51,6 +51,6 @@ public class TimeParseOperatorConversion implements SqlOperatorConversion
final RexNode rexNode
)
{
return OperatorConversions.functionCall(plannerContext, rowSignature, rexNode, "timestamp_parse");
return OperatorConversions.convertCall(plannerContext, rowSignature, rexNode, "timestamp_parse");
}
}

2
sql/src/main/java/io/druid/sql/calcite/expression/TimeShiftOperatorConversion.java
View File

@ -51,6 +51,6 @@ public class TimeShiftOperatorConversion implements SqlOperatorConversion
final RexNode rexNode
)
{
return OperatorConversions.functionCall(plannerContext, rowSignature, rexNode, "timestamp_shift");
return OperatorConversions.convertCall(plannerContext, rowSignature, rexNode, "timestamp_shift");
}
}

101
sql/src/main/java/io/druid/sql/calcite/expression/TrimOperatorConversion.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.sql.calcite.expression;
import com.google.common.collect.ImmutableList;
import io.druid.sql.calcite.planner.PlannerContext;
import io.druid.sql.calcite.table.RowSignature;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.fun.SqlTrimFunction;
import javax.annotation.Nullable;
public class TrimOperatorConversion implements SqlOperatorConversion
{
@Nullable
public static DruidExpression makeTrimExpression(
final SqlTrimFunction.Flag trimStyle,
final DruidExpression stringExpression,
final DruidExpression charsExpression
)
{
final String functionName;
switch (trimStyle) {
case LEADING:
functionName = "ltrim";
break;
case TRAILING:
functionName = "rtrim";
break;
case BOTH:
functionName = "trim";
break;
default:
// Not reached
throw new UnsupportedOperationException();
}
// Druid version of trim is multi-function (ltrim/rtrim/trim) and the other two args are swapped.
return DruidExpression.fromFunctionCall(functionName, ImmutableList.of(stringExpression, charsExpression));
}
@Override
public SqlOperator calciteOperator()
{
return SqlStdOperatorTable.TRIM;
}
@Override
public DruidExpression toDruidExpression(
final PlannerContext plannerContext,
final RowSignature rowSignature,
final RexNode rexNode
)
{
// TRIM(<style> <chars> FROM <arg>)
final RexCall call = (RexCall) rexNode;
final RexLiteral flag = (RexLiteral) call.getOperands().get(0);
final SqlTrimFunction.Flag trimStyle = (SqlTrimFunction.Flag) flag.getValue();
final DruidExpression charsExpression = Expressions.toDruidExpression(
plannerContext,
rowSignature,
call.getOperands().get(1)
);
final DruidExpression stringExpression = Expressions.toDruidExpression(
plannerContext,
rowSignature,
call.getOperands().get(2)
);
if (charsExpression == null || stringExpression == null) {
return null;
}
return makeTrimExpression(trimStyle, stringExpression, charsExpression);
}
}

2
sql/src/main/java/io/druid/sql/calcite/filtration/Filtration.java
View File

@ -21,8 +21,8 @@ package io.druid.sql.calcite.filtration;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import io.druid.java.util.common.Intervals;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.Intervals;
import io.druid.math.expr.ExprMacroTable;
import io.druid.query.filter.DimFilter;
import io.druid.query.filter.ExpressionDimFilter;

8
sql/src/main/java/io/druid/sql/guice/SqlModule.java
View File

@ -35,11 +35,14 @@ import io.druid.sql.avatica.AvaticaServerConfig;
import io.druid.sql.avatica.DruidAvaticaHandler;
import io.druid.sql.calcite.aggregation.ApproxCountDistinctSqlAggregator;
import io.druid.sql.calcite.aggregation.SqlAggregator;
import io.druid.sql.calcite.expression.BTrimOperatorConversion;
import io.druid.sql.calcite.expression.CeilOperatorConversion;
import io.druid.sql.calcite.expression.ExtractOperatorConversion;
import io.druid.sql.calcite.expression.FloorOperatorConversion;
import io.druid.sql.calcite.expression.LTrimOperatorConversion;
import io.druid.sql.calcite.expression.LookupOperatorConversion;
import io.druid.sql.calcite.expression.MillisToTimestampOperatorConversion;
import io.druid.sql.calcite.expression.RTrimOperatorConversion;
import io.druid.sql.calcite.expression.RegexpExtractOperatorConversion;
import io.druid.sql.calcite.expression.SqlOperatorConversion;
import io.druid.sql.calcite.expression.SubstringOperatorConversion;
@ -50,6 +53,7 @@ import io.druid.sql.calcite.expression.TimeFormatOperatorConversion;
import io.druid.sql.calcite.expression.TimeParseOperatorConversion;
import io.druid.sql.calcite.expression.TimeShiftOperatorConversion;
import io.druid.sql.calcite.expression.TimestampToMillisOperatorConversion;
import io.druid.sql.calcite.expression.TrimOperatorConversion;
import io.druid.sql.calcite.planner.Calcites;
import io.druid.sql.calcite.planner.PlannerConfig;
import io.druid.sql.calcite.schema.DruidSchema;
@ -82,6 +86,10 @@ public class SqlModule implements Module
.add(TimeParseOperatorConversion.class)
.add(TimeShiftOperatorConversion.class)
.add(TimestampToMillisOperatorConversion.class)
.add(TrimOperatorConversion.class)
.add(BTrimOperatorConversion.class)
.add(LTrimOperatorConversion.class)
.add(RTrimOperatorConversion.class)
.build();
private static final String PROPERTY_SQL_ENABLE = "druid.sql.enable";

87
sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java
View File

@ -45,6 +45,7 @@ import io.druid.query.aggregation.FloatMinAggregatorFactory;
import io.druid.query.aggregation.LongMaxAggregatorFactory;
import io.druid.query.aggregation.LongMinAggregatorFactory;
import io.druid.query.aggregation.LongSumAggregatorFactory;
import io.druid.query.aggregation.PostAggregator;
import io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory;
import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory;
import io.druid.query.aggregation.post.ArithmeticPostAggregator;
@ -250,6 +251,56 @@ public class CalciteQueryTest
);
}
@Test
public void testSelectTrimFamily() throws Exception
{
// TRIM has some whacky parsing. Make sure the different forms work.
testQuery(
"SELECT\n"
+ "TRIM(BOTH 'x' FROM 'xfoox'),\n"
+ "TRIM(TRAILING 'x' FROM 'xfoox'),\n"
+ "TRIM(' ' FROM ' foo '),\n"
+ "TRIM(TRAILING FROM ' foo '),\n"
+ "TRIM(' foo '),\n"
+ "BTRIM(' foo '),\n"
+ "BTRIM('xfoox', 'x'),\n"
+ "LTRIM(' foo '),\n"
+ "LTRIM('xfoox', 'x'),\n"
+ "RTRIM(' foo '),\n"
+ "RTRIM('xfoox', 'x'),\n"
+ "COUNT(*)\n"
+ "FROM foo",
ImmutableList.of(
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(QSS(Filtration.eternity()))
.granularity(Granularities.ALL)
.aggregators(AGGS(new CountAggregatorFactory("a0")))
.postAggregators(
ImmutableList.<PostAggregator>builder()
.add(EXPRESSION_POST_AGG("p0", "'foo'"))
.add(EXPRESSION_POST_AGG("p1", "'xfoo'"))
.add(EXPRESSION_POST_AGG("p2", "'foo'"))
.add(EXPRESSION_POST_AGG("p3", "' foo'"))
.add(EXPRESSION_POST_AGG("p4", "'foo'"))
.add(EXPRESSION_POST_AGG("p5", "'foo'"))
.add(EXPRESSION_POST_AGG("p6", "'foo'"))
.add(EXPRESSION_POST_AGG("p7", "'foo '"))
.add(EXPRESSION_POST_AGG("p8", "'foox'"))
.add(EXPRESSION_POST_AGG("p9", "' foo'"))
.add(EXPRESSION_POST_AGG("p10", "'xfoo'"))
.build()
)
.context(TIMESERIES_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{"foo", "xfoo", "foo", " foo", "foo", "foo", "foo", "foo ", "foox", " foo", "xfoo", 6L}
)
);
}
@Test
public void testExplainSelectConstantExpression() throws Exception
{
@ -1213,7 +1264,6 @@ public class CalciteQueryTest
final List<String> queries = ImmutableList.of(
"SELECT dim1 FROM druid.foo ORDER BY dim1", // SELECT query with order by
"SELECT TRIM(dim1) FROM druid.foo", // TRIM function
"SELECT COUNT(*) FROM druid.foo x, druid.foo y", // Self-join
"SELECT DISTINCT dim2 FROM druid.foo ORDER BY dim2 LIMIT 2 OFFSET 5" // DISTINCT with OFFSET
);
@ -3799,6 +3849,41 @@ public class CalciteQueryTest
);
}
@Test
public void testCountDistinctOfTrim() throws Exception
{
// Test a couple different syntax variants of TRIM.
testQuery(
"SELECT COUNT(DISTINCT TRIM(BOTH ' ' FROM dim1)) FROM druid.foo WHERE TRIM(dim1) <> ''",
ImmutableList.of(
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(QSS(Filtration.eternity()))
.filters(NOT(SELECTOR("dim1", "", null)))
.granularity(Granularities.ALL)
.virtualColumns(EXPRESSION_VIRTUAL_COLUMN("a0:v", "trim(\"dim1\",' ')", ValueType.STRING))
.filters(EXPRESSION_FILTER("(trim(\"dim1\",' ') != '')"))
.aggregators(
AGGS(
new CardinalityAggregatorFactory(
"a0",
null,
DIMS(new DefaultDimensionSpec("a0:v", "a0:v", ValueType.STRING)),
false,
true
)
)
)
.context(TIMESERIES_CONTEXT_DEFAULT)
.build()
),
ImmutableList.of(
new Object[]{5L}
)
);
}
@Test
public void testSillyQuarters() throws Exception
{

40
sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java
View File

@ -43,6 +43,7 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.fun.SqlTrimFunction;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.SqlTypeName;
import org.joda.time.DateTime;
@ -72,6 +73,7 @@ public class ExpressionsTest
.add("x", ValueType.FLOAT)
.add("y", ValueType.LONG)
.add("s", ValueType.STRING)
.add("spacey", ValueType.STRING)
.add("tstr", ValueType.STRING)
.add("dstr", ValueType.STRING)
.build();
@ -82,6 +84,7 @@ public class ExpressionsTest
.put("x", 2.5)
.put("y", 3.0)
.put("s", "foo")
.put("spacey", " hey there ")
.put("tstr", "2000-02-03 04:05:06")
.put("dstr", "2000-02-03")
.build();
@ -150,6 +153,43 @@ public class ExpressionsTest
);
}
@Test
public void testTrim()
{
testExpression(
rexBuilder.makeCall(
SqlStdOperatorTable.TRIM,
rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH),
rexBuilder.makeLiteral(" "),
inputRef("spacey")
),
DruidExpression.fromExpression("trim(\"spacey\",' ')"),
"hey there"
);
testExpression(
rexBuilder.makeCall(
SqlStdOperatorTable.TRIM,
rexBuilder.makeFlag(SqlTrimFunction.Flag.LEADING),
rexBuilder.makeLiteral(" h"),
inputRef("spacey")
),
DruidExpression.fromExpression("ltrim(\"spacey\",' h')"),
"ey there "
);
testExpression(
rexBuilder.makeCall(
SqlStdOperatorTable.TRIM,
rexBuilder.makeFlag(SqlTrimFunction.Flag.TRAILING),
rexBuilder.makeLiteral(" e"),
inputRef("spacey")
),
DruidExpression.fromExpression("rtrim(\"spacey\",' e')"),
" hey ther"
);
}
@Test
public void testTimeFloor()
{

11
sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java
View File

@ -209,13 +209,18 @@ public class SqlResourceTest
@Test
public void testCannotConvert() throws Exception
{
// TRIM unsupported
final QueryInterruptedException exception = doPost(new SqlQuery("SELECT TRIM(dim1) FROM druid.foo", null)).lhs;
// SELECT + ORDER unsupported
final QueryInterruptedException exception = doPost(
new SqlQuery("SELECT dim1 FROM druid.foo ORDER BY dim1", null)
).lhs;
Assert.assertNotNull(exception);
Assert.assertEquals(QueryInterruptedException.UNKNOWN_EXCEPTION, exception.getErrorCode());
Assert.assertEquals(ISE.class.getName(), exception.getErrorClass());
Assert.assertTrue(exception.getMessage().contains("Cannot build plan for query: SELECT TRIM(dim1) FROM druid.foo"));
Assert.assertTrue(
exception.getMessage()
.contains("Cannot build plan for query: SELECT dim1 FROM druid.foo ORDER BY dim1")
);
}
@Test