From a44ab109d55ab52366386d99cd116db359ebdd43 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 17 Dec 2024 18:49:16 -0800 Subject: [PATCH] remove druid.expressions.useStrictBooleans in favor of always being true (#17568) --- .../InputSourceSamplerDiscoveryTest.java | 66 --- .../druid/common/config/NullHandling.java | 4 +- .../druid/math/expr/BinaryEvalOpExprBase.java | 10 +- .../math/expr/BinaryLogicalOperatorExpr.java | 20 +- .../org/apache/druid/math/expr/ExprEval.java | 33 +- .../druid/math/expr/ExpressionProcessing.java | 15 - .../math/expr/ExpressionProcessingConfig.java | 17 +- .../druid/math/expr/UnaryOperatorExpr.java | 13 - .../vector/VectorComparisonProcessors.java | 467 ----------------- .../math/expr/vector/VectorProcessors.java | 26 +- .../druid/segment/filter/NotFilter.java | 10 +- .../org/apache/druid/math/expr/EvalTest.java | 468 +++++------------- .../druid/math/expr/OutputTypeTest.java | 106 ++-- .../query/scan/NestedDataScanQueryTest.java | 71 --- ...ExpressionFilterNonStrictBooleansTest.java | 143 ------ .../segment/filter/ExpressionFilterTest.java | 17 +- .../segment/transform/TransformSpecTest.java | 78 +-- .../calcite/rule/DruidLogicalValuesRule.java | 3 +- .../rule/DruidLogicalValuesRuleTest.java | 3 +- 19 files changed, 206 insertions(+), 1364 deletions(-) delete mode 100644 processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java index 0220aacd892..463416fc33e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java @@ -31,7 +31,6 @@ import org.apache.druid.data.input.impl.LongDimensionSchema; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.segment.AutoTypeColumnSchema; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; @@ -56,71 +55,6 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest ); private InputSourceSampler inputSourceSampler = new InputSourceSampler(OBJECT_MAPPER); - @Test - public void testDiscoveredTypesNonStrictBooleans() - { - - try { - ExpressionProcessing.initializeForStrictBooleansTests(false); - final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n')); - final SamplerResponse response = inputSourceSampler.sample( - inputSource, - new JsonInputFormat(null, null, null, null, null), - DataSchema.builder() - .withDataSource("test") - .withTimestamp(new TimestampSpec("t", null, null)) - .withDimensions(DimensionsSpec.builder().useSchemaDiscovery(true).build()) - .build(), - null - ); - - Assert.assertEquals(6, response.getNumRowsRead()); - Assert.assertEquals(5, response.getNumRowsIndexed()); - Assert.assertEquals(6, response.getData().size()); - Assert.assertEquals( - ImmutableList.of( - new StringDimensionSchema("string"), - new LongDimensionSchema("long"), - new DoubleDimensionSchema("double"), - new StringDimensionSchema("bool"), - new StringDimensionSchema("variant"), - new AutoTypeColumnSchema("array", null), - new AutoTypeColumnSchema("nested", null) - ), - response.getLogicalDimensions() - ); - - Assert.assertEquals( - ImmutableList.of( - new AutoTypeColumnSchema("string", null), - new AutoTypeColumnSchema("long", null), - new AutoTypeColumnSchema("double", null), - new AutoTypeColumnSchema("bool", null), - new AutoTypeColumnSchema("variant", null), - new AutoTypeColumnSchema("array", null), - new AutoTypeColumnSchema("nested", null) - ), - response.getPhysicalDimensions() - ); - Assert.assertEquals( - RowSignature.builder() - .addTimeColumn() - .add("string", ColumnType.STRING) - .add("long", ColumnType.LONG) - .add("double", ColumnType.DOUBLE) - .add("bool", ColumnType.STRING) - .add("variant", ColumnType.STRING) - .add("array", ColumnType.LONG_ARRAY) - .add("nested", ColumnType.NESTED_DATA) - .build(), - response.getLogicalSegmentSchema() - ); - } - finally { - ExpressionProcessing.initializeForTests(); - } - } - @Test public void testDiscoveredTypesStrictBooleans() { diff --git a/processing/src/main/java/org/apache/druid/common/config/NullHandling.java b/processing/src/main/java/org/apache/druid/common/config/NullHandling.java index b98d81421d2..578c64acbc5 100644 --- a/processing/src/main/java/org/apache/druid/common/config/NullHandling.java +++ b/processing/src/main/java/org/apache/druid/common/config/NullHandling.java @@ -22,7 +22,6 @@ package org.apache.druid.common.config; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.inject.Inject; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.filter.ValueMatcher; @@ -130,8 +129,7 @@ public class NullHandling public static boolean useThreeValueLogic() { return sqlCompatible() && - INSTANCE.isUseThreeValueLogicForNativeFilters() && - ExpressionProcessing.useStrictBooleans(); + INSTANCE.isUseThreeValueLogicForNativeFilters(); } @Nullable diff --git a/processing/src/main/java/org/apache/druid/math/expr/BinaryEvalOpExprBase.java b/processing/src/main/java/org/apache/druid/math/expr/BinaryEvalOpExprBase.java index 2104dcc45db..6b7bb21a1f7 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/BinaryEvalOpExprBase.java +++ b/processing/src/main/java/org/apache/druid/math/expr/BinaryEvalOpExprBase.java @@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.segment.column.Types; import javax.annotation.Nullable; import java.util.Objects; @@ -206,9 +205,6 @@ abstract class BinaryBooleanOpExprBase extends BinaryOpExprBase result = evalDouble(leftVal.asDouble(), rightVal.asDouble()); break; } - if (!ExpressionProcessing.useStrictBooleans() && !type.is(ExprType.STRING) && !type.isArray()) { - return ExprEval.ofBoolean(result, type); - } return ExprEval.ofLongBoolean(result); } @@ -224,11 +220,7 @@ abstract class BinaryBooleanOpExprBase extends BinaryOpExprBase @Override public ExpressionType getOutputType(InputBindingInspector inspector) { - ExpressionType implicitCast = super.getOutputType(inspector); - if (ExpressionProcessing.useStrictBooleans() || Types.isNullOr(implicitCast, ExprType.STRING)) { - return ExpressionType.LONG; - } - return implicitCast; + return ExpressionType.LONG; } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java b/processing/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java index 13bb4e7f52f..880271ff1de 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java +++ b/processing/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java @@ -340,9 +340,6 @@ class BinAndExpr extends BinaryOpExprBase public ExprEval eval(ObjectBinding bindings) { ExprEval leftVal = left.eval(bindings); - if (!ExpressionProcessing.useStrictBooleans()) { - return leftVal.asBoolean() ? right.eval(bindings) : leftVal; - } // if left is false, always false if (leftVal.value() != null && !leftVal.asBoolean()) { @@ -376,9 +373,7 @@ class BinAndExpr extends BinaryOpExprBase @Override public boolean canVectorize(InputBindingInspector inspector) { - return ExpressionProcessing.useStrictBooleans() && - inspector.areSameTypes(left, right) && - inspector.canVectorize(left, right); + return inspector.areSameTypes(left, right) && inspector.canVectorize(left, right); } @Override @@ -391,9 +386,6 @@ class BinAndExpr extends BinaryOpExprBase @Override public ExpressionType getOutputType(InputBindingInspector inspector) { - if (!ExpressionProcessing.useStrictBooleans()) { - return super.getOutputType(inspector); - } return ExpressionType.LONG; } } @@ -415,9 +407,6 @@ class BinOrExpr extends BinaryOpExprBase public ExprEval eval(ObjectBinding bindings) { ExprEval leftVal = left.eval(bindings); - if (!ExpressionProcessing.useStrictBooleans()) { - return leftVal.asBoolean() ? leftVal : right.eval(bindings); - } // if left is true, always true if (leftVal.value() != null && leftVal.asBoolean()) { @@ -454,9 +443,7 @@ class BinOrExpr extends BinaryOpExprBase public boolean canVectorize(InputBindingInspector inspector) { - return ExpressionProcessing.useStrictBooleans() && - inspector.areSameTypes(left, right) && - inspector.canVectorize(left, right); + return inspector.areSameTypes(left, right) && inspector.canVectorize(left, right); } @Override @@ -469,9 +456,6 @@ class BinOrExpr extends BinaryOpExprBase @Override public ExpressionType getOutputType(InputBindingInspector inspector) { - if (!ExpressionProcessing.useStrictBooleans()) { - return super.getOutputType(inspector); - } return ExpressionType.LONG; } } diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java index a18ca8b61e2..3c2e5630df4 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -242,11 +242,7 @@ public abstract class ExprEval if (Number.class.isAssignableFrom(next) || next == String.class || next == Boolean.class) { // coerce booleans if (next == Boolean.class) { - if (ExpressionProcessing.useStrictBooleans()) { - next = Long.class; - } else { - next = String.class; - } + next = Long.class; } if (existing == null) { return next; @@ -350,28 +346,6 @@ public abstract class ExprEval return new ArrayExprEval(outputType, value); } - /** - * Convert a boolean back into native expression type - * - * Do not use this method unless {@link ExpressionProcessing#useStrictBooleans()} is set to false. - * {@link ExpressionType#LONG} is the Druid boolean unless this mode is enabled, so use {@link #ofLongBoolean} - * instead. - */ - @Deprecated - public static ExprEval ofBoolean(boolean value, ExpressionType type) - { - switch (type.getType()) { - case DOUBLE: - return of(Evals.asDouble(value)); - case LONG: - return ofLongBoolean(value); - case STRING: - return of(String.valueOf(value)); - default: - throw new Types.InvalidCastBooleanException(type); - } - } - /** * Convert a boolean into a long expression type */ @@ -421,10 +395,7 @@ public abstract class ExprEval return new LongExprEval((Number) val); } if (val instanceof Boolean) { - if (ExpressionProcessing.useStrictBooleans()) { - return ofLongBoolean((Boolean) val); - } - return new StringExprEval(String.valueOf(val)); + return ofLongBoolean((Boolean) val); } if (val instanceof Long[]) { final Long[] inputArray = (Long[]) val; diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java index 9a4d2ef4694..2387cea909f 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java @@ -48,12 +48,6 @@ public class ExpressionProcessing INSTANCE = new ExpressionProcessingConfig(null, null, null, null); } - @VisibleForTesting - public static void initializeForStrictBooleansTests(boolean useStrict) - { - INSTANCE = new ExpressionProcessingConfig(useStrict, null, null, null); - } - @VisibleForTesting public static void initializeForHomogenizeNullMultiValueStrings() { @@ -66,15 +60,6 @@ public class ExpressionProcessing INSTANCE = new ExpressionProcessingConfig(null, null, null, true); } - /** - * All boolean expressions are {@link ExpressionType#LONG} - */ - public static boolean useStrictBooleans() - { - checkInitialized(); - return INSTANCE.isUseStrictBooleans(); - } - /** * All {@link ExprType#ARRAY} values will be converted to {@link ExpressionType#STRING} by their column selectors * (not within expression processing) to be treated as multi-value strings instead of native arrays. diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java index 3d235fe3dda..a0d58eb4806 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java @@ -30,6 +30,7 @@ public class ExpressionProcessingConfig { private static final Logger LOG = new Logger(ExpressionProcessingConfig.class); + @Deprecated public static final String NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING = "druid.expressions.useStrictBooleans"; // Coerce arrays to multi value strings public static final String PROCESS_ARRAYS_AS_MULTIVALUE_STRINGS_CONFIG_STRING = @@ -39,9 +40,6 @@ public class ExpressionProcessingConfig "druid.expressions.homogenizeNullMultiValueStringArrays"; public static final String ALLOW_VECTORIZE_FALLBACK = "druid.expressions.allowVectorizeFallback"; - @JsonProperty("useStrictBooleans") - private final boolean useStrictBooleans; - @JsonProperty("processArraysAsMultiValueStrings") private final boolean processArraysAsMultiValueStrings; @@ -51,9 +49,13 @@ public class ExpressionProcessingConfig @JsonProperty("allowVectorizeFallback") private final boolean allowVectorizeFallback; + @Deprecated + @JsonProperty("useStrictBooleans") + private final boolean useStrictBooleans; + @JsonCreator public ExpressionProcessingConfig( - @JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans, + @Deprecated @JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans, @JsonProperty("processArraysAsMultiValueStrings") @Nullable Boolean processArraysAsMultiValueStrings, @JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays, @JsonProperty("allowVectorizeFallback") @Nullable Boolean allowVectorizeFallback @@ -83,17 +85,12 @@ public class ExpressionProcessingConfig final String docsBaseFormat = "https://druid.apache.org/docs/%s/querying/sql-data-types#%s"; if (!this.useStrictBooleans) { LOG.warn( - "druid.expressions.useStrictBooleans set to 'false', we recommend using 'true' if using SQL to query Druid for the most SQL compliant behavior, see %s for details", + "druid.expressions.useStrictBooleans set to 'false', but has been removed from Druid and is always 'true' now for the most SQL compliant behavior, see %s for details", StringUtils.format(docsBaseFormat, version, "boolean-logic") ); } } - public boolean isUseStrictBooleans() - { - return useStrictBooleans; - } - public boolean processArraysAsMultiValueStrings() { return processArraysAsMultiValueStrings; diff --git a/processing/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java b/processing/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java index f9f2c5bbcc2..30b3f9f4fc7 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java +++ b/processing/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java @@ -26,7 +26,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.math.expr.vector.VectorMathProcessors; import org.apache.druid.math.expr.vector.VectorProcessors; -import org.apache.druid.segment.column.Types; import javax.annotation.Nullable; import java.math.BigInteger; @@ -181,11 +180,6 @@ class UnaryNotExpr extends UnaryExpr if (NullHandling.sqlCompatible() && (ret.value() == null)) { return ExprEval.of(null); } - if (!ExpressionProcessing.useStrictBooleans()) { - // conforming to other boolean-returning binary operators - ExpressionType retType = ret.type().is(ExprType.DOUBLE) ? ExpressionType.DOUBLE : ExpressionType.LONG; - return ExprEval.ofBoolean(!ret.asBoolean(), retType); - } return ExprEval.ofLongBoolean(!ret.asBoolean()); } @@ -193,13 +187,6 @@ class UnaryNotExpr extends UnaryExpr @Override public ExpressionType getOutputType(InputBindingInspector inspector) { - if (!ExpressionProcessing.useStrictBooleans()) { - ExpressionType implicitCast = super.getOutputType(inspector); - if (Types.is(implicitCast, ExprType.STRING)) { - return ExpressionType.LONG; - } - return implicitCast; - } return ExpressionType.LONG; } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java index a132c0ee672..1df76f2bed2 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java @@ -23,7 +23,6 @@ import org.apache.druid.java.util.common.guava.Comparators; import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.segment.column.Types; @@ -33,50 +32,6 @@ import java.util.function.Supplier; public class VectorComparisonProcessors { - @Deprecated - public static ExprVectorProcessor makeComparisonProcessor( - Expr.VectorInputBindingInspector inspector, - Expr left, - Expr right, - Supplier longOutStringsInFunctionVectorProcessor, - Supplier longOutLongsInProcessor, - Supplier doubleOutLongDoubleInProcessor, - Supplier doubleOutDoubleLongInProcessor, - Supplier doubleOutDoublesInProcessor - ) - { - assert !ExpressionProcessing.useStrictBooleans(); - final ExpressionType leftType = left.getOutputType(inspector); - final ExpressionType rightType = right.getOutputType(inspector); - ExprVectorProcessor processor = null; - if (Types.is(leftType, ExprType.STRING)) { - if (Types.isNullOr(rightType, ExprType.STRING)) { - processor = longOutStringsInFunctionVectorProcessor.get(); - } else { - processor = doubleOutDoublesInProcessor.get(); - } - } else if (leftType == null) { - if (Types.isNullOr(rightType, ExprType.STRING)) { - processor = longOutStringsInFunctionVectorProcessor.get(); - } - } else if (leftType.is(ExprType.DOUBLE) || Types.is(rightType, ExprType.DOUBLE)) { - processor = doubleOutDoublesInProcessor.get(); - } - if (processor != null) { - return (ExprVectorProcessor) processor; - } - // fall through to normal math processor logic - return VectorMathProcessors.makeMathProcessor( - inspector, - left, - right, - longOutLongsInProcessor, - doubleOutLongDoubleInProcessor, - doubleOutDoubleLongInProcessor, - doubleOutDoublesInProcessor - ); - } - public static ExprVectorProcessor makeBooleanProcessor( Expr.VectorInputBindingInspector inspector, Expr left, @@ -131,75 +86,6 @@ public class VectorComparisonProcessors Expr right ) { - if (!ExpressionProcessing.useStrictBooleans()) { - return makeComparisonProcessor( - inspector, - left, - right, - () -> new LongOutObjectsInFunctionVectorProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize(), - ExpressionType.STRING - ) - { - @Nullable - @Override - Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) - { - return Evals.asLong(Objects.equals(leftVal, rightVal)); - } - }, - () -> new LongOutLongsInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left == right); - } - }, - () -> new DoubleOutLongDoubleInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(left == right); - } - }, - () -> new DoubleOutDoubleLongInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(left == right); - } - }, - () -> new DoubleOutDoublesInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(left == right); - } - } - ); - } return makeBooleanProcessor( inspector, left, @@ -275,75 +161,6 @@ public class VectorComparisonProcessors Expr right ) { - if (!ExpressionProcessing.useStrictBooleans()) { - return makeComparisonProcessor( - inspector, - left, - right, - () -> new LongOutObjectsInFunctionVectorProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize(), - ExpressionType.STRING - ) - { - @Nullable - @Override - Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) - { - return Evals.asLong(!Objects.equals(leftVal, rightVal)); - } - }, - () -> new LongOutLongsInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left != right); - } - }, - () -> new DoubleOutLongDoubleInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(left != right); - } - }, - () -> new DoubleOutDoubleLongInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(left != right); - } - }, - () -> new DoubleOutDoublesInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(left != right); - } - } - ); - } return makeBooleanProcessor( inspector, left, @@ -419,77 +236,6 @@ public class VectorComparisonProcessors Expr right ) { - if (!ExpressionProcessing.useStrictBooleans()) { - return makeComparisonProcessor( - inspector, - left, - right, - () -> new LongOutObjectsInFunctionVectorProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize(), - ExpressionType.STRING - ) - { - @Nullable - @Override - Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) - { - return Evals.asLong( - Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) >= 0 - ); - } - }, - () -> new LongOutLongsInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left >= right); - } - }, - () -> new DoubleOutLongDoubleInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(Double.compare(left, right) >= 0); - } - }, - () -> new DoubleOutDoubleLongInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(Double.compare(left, right) >= 0); - } - }, - () -> new DoubleOutDoublesInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(Double.compare(left, right) >= 0); - } - } - ); - } return makeBooleanProcessor( inspector, left, @@ -567,77 +313,6 @@ public class VectorComparisonProcessors Expr right ) { - if (!ExpressionProcessing.useStrictBooleans()) { - return makeComparisonProcessor( - inspector, - left, - right, - () -> new LongOutObjectsInFunctionVectorProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize(), - ExpressionType.STRING - ) - { - @Nullable - @Override - Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) - { - return Evals.asLong( - Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) > 0 - ); - } - }, - () -> new LongOutLongsInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left > right); - } - }, - () -> new DoubleOutLongDoubleInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(Double.compare(left, right) > 0); - } - }, - () -> new DoubleOutDoubleLongInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(Double.compare(left, right) > 0); - } - }, - () -> new DoubleOutDoublesInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(Double.compare(left, right) > 0); - } - } - ); - } return makeBooleanProcessor( inspector, left, @@ -715,77 +390,6 @@ public class VectorComparisonProcessors Expr right ) { - if (!ExpressionProcessing.useStrictBooleans()) { - return makeComparisonProcessor( - inspector, - left, - right, - () -> new LongOutObjectsInFunctionVectorProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize(), - ExpressionType.STRING - ) - { - @Nullable - @Override - Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) - { - return Evals.asLong( - Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) <= 0 - ); - } - }, - () -> new LongOutLongsInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left <= right); - } - }, - () -> new DoubleOutLongDoubleInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(Double.compare(left, right) <= 0); - } - }, - () -> new DoubleOutDoubleLongInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(Double.compare(left, right) <= 0); - } - }, - () -> new DoubleOutDoublesInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(Double.compare(left, right) <= 0); - } - } - ); - } return makeBooleanProcessor( inspector, left, @@ -863,77 +467,6 @@ public class VectorComparisonProcessors Expr right ) { - if (!ExpressionProcessing.useStrictBooleans()) { - return makeComparisonProcessor( - inspector, - left, - right, - () -> new LongOutObjectsInFunctionVectorProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize(), - ExpressionType.STRING - ) - { - @Nullable - @Override - Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) - { - return Evals.asLong( - Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) < 0 - ); - } - }, - () -> new LongOutLongsInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left < right); - } - }, - () -> new DoubleOutLongDoubleInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(Double.compare(left, right) < 0); - } - }, - () -> new DoubleOutDoubleLongInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(Double.compare(left, right) < 0); - } - }, - () -> new DoubleOutDoublesInFunctionVectorValueProcessor( - left.asVectorProcessor(inspector), - right.asVectorProcessor(inspector), - inspector.getMaxVectorSize() - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(Double.compare(left, right) < 0); - } - } - ); - } return makeBooleanProcessor( inspector, left, diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java index d5d3933b860..1f5727b8361 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -25,7 +25,6 @@ import org.apache.druid.error.DruidException; import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.Exprs; import org.apache.druid.segment.column.Types; @@ -658,25 +657,14 @@ public class VectorProcessors } }; } else if (Types.is(inputType, ExprType.DOUBLE)) { - if (!ExpressionProcessing.useStrictBooleans()) { - processor = new DoubleOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize) + processor = new LongOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize) + { + @Override + public long apply(double input) { - @Override - public double apply(double input) - { - return Evals.asDouble(!Evals.asBoolean(input)); - } - }; - } else { - processor = new LongOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize) - { - @Override - public long apply(double input) - { - return Evals.asLong(!Evals.asBoolean(input)); - } - }; - } + return Evals.asLong(!Evals.asBoolean(input)); + } + }; } if (processor == null) { throw Exprs.cannotVectorize(); diff --git a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java index 3e89148e2cc..e44cf2b57e4 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java @@ -21,7 +21,6 @@ package org.apache.druid.segment.filter; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.filter.ColumnIndexSelector; import org.apache.druid.query.filter.Filter; @@ -45,11 +44,10 @@ import java.util.Set; /** * Nice filter you have there... NOT! * - * If {@link ExpressionProcessing#useStrictBooleans()} and {@link NullHandling#sqlCompatible()} are both true, this - * filter inverts the {@code includeUnknown} flag to properly map Druids native two-valued logic (true, false) to SQL - * three-valued logic (true, false, unknown). At the top level, this flag is always passed in as 'false', and is only - * flipped by this filter. Other logical filters ({@link AndFilter} and {@link OrFilter}) propagate the value of - * {@code includeUnknown} to their children. + * If {@link NullHandling#sqlCompatible()} is true, this filter inverts the {@code includeUnknown} flag to properly + * map Druids native two-valued logic (true, false) to SQL three-valued logic (true, false, unknown). At the top level, + * this flag is always passed in as 'false', and is only flipped by this filter. Other logical filters + * ({@link AndFilter} and {@link OrFilter}) propagate the value of {@code includeUnknown} to their children. * * For example, if the base filter is equality, by default value matchers and indexes only return true for the rows * that are equal to the value. When wrapped in a not filter, the not filter indicates that the equality matchers and diff --git a/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java b/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java index 2f68840955f..ae515d13abf 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java @@ -41,7 +41,6 @@ import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; /** */ @@ -84,93 +83,50 @@ public class EvalTest extends InitializedNullHandlingTest assertEquals(2.0, evalDouble("\"x\"", bindings), 0.0001); assertEquals(304.0, evalDouble("300 + \"x\" * 2", bindings), 0.0001); - try { - ExpressionProcessing.initializeForStrictBooleansTests(false); - Assert.assertFalse(evalDouble("1.0 && 0.0", bindings) > 0.0); - Assert.assertTrue(evalDouble("1.0 && 2.0", bindings) > 0.0); + Assert.assertEquals(0L, evalLong("1.0 && 0.0", bindings)); + Assert.assertEquals(1L, evalLong("1.0 && 2.0", bindings)); - Assert.assertTrue(evalDouble("1.0 || 0.0", bindings) > 0.0); - Assert.assertFalse(evalDouble("0.0 || 0.0", bindings) > 0.0); + Assert.assertEquals(1L, evalLong("1.0 || 0.0", bindings)); + Assert.assertEquals(0L, evalLong("0.0 || 0.0", bindings)); - Assert.assertTrue(evalDouble("2.0 > 1.0", bindings) > 0.0); - Assert.assertTrue(evalDouble("2.0 >= 2.0", bindings) > 0.0); - Assert.assertTrue(evalDouble("1.0 < 2.0", bindings) > 0.0); - Assert.assertTrue(evalDouble("2.0 <= 2.0", bindings) > 0.0); - Assert.assertTrue(evalDouble("2.0 == 2.0", bindings) > 0.0); - Assert.assertTrue(evalDouble("2.0 != 1.0", bindings) > 0.0); + Assert.assertEquals(1L, evalLong("2.0 > 1.0", bindings)); + Assert.assertEquals(1L, evalLong("2.0 >= 2.0", bindings)); + Assert.assertEquals(1L, evalLong("1.0 < 2.0", bindings)); + Assert.assertEquals(1L, evalLong("2.0 <= 2.0", bindings)); + Assert.assertEquals(1L, evalLong("2.0 == 2.0", bindings)); + Assert.assertEquals(1L, evalLong("2.0 != 1.0", bindings)); - Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings)); - Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings)); - Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings)); - Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings)); + Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings)); + Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings)); + Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings)); + Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings)); - Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings)); - Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings)); - Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings)); - Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings)); + Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings)); + Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings)); + Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings)); + Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings)); - Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings)); - Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings)); - Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings)); - Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings)); + Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings)); + Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings)); + Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings)); + Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings)); - Assert.assertTrue(evalDouble("!-1.0", bindings) > 0.0); - Assert.assertTrue(evalDouble("!0.0", bindings) > 0.0); - Assert.assertFalse(evalDouble("!2.0", bindings) > 0.0); - } - finally { - ExpressionProcessing.initializeForTests(); - } - try { - ExpressionProcessing.initializeForStrictBooleansTests(true); - Assert.assertEquals(0L, evalLong("1.0 && 0.0", bindings)); - Assert.assertEquals(1L, evalLong("1.0 && 2.0", bindings)); + Assert.assertEquals(1L, evalLong("!-1.0", bindings)); + Assert.assertEquals(1L, evalLong("!0.0", bindings)); + Assert.assertEquals(0L, evalLong("!2.0", bindings)); - Assert.assertEquals(1L, evalLong("1.0 || 0.0", bindings)); - Assert.assertEquals(0L, evalLong("0.0 || 0.0", bindings)); - - Assert.assertEquals(1L, evalLong("2.0 > 1.0", bindings)); - Assert.assertEquals(1L, evalLong("2.0 >= 2.0", bindings)); - Assert.assertEquals(1L, evalLong("1.0 < 2.0", bindings)); - Assert.assertEquals(1L, evalLong("2.0 <= 2.0", bindings)); - Assert.assertEquals(1L, evalLong("2.0 == 2.0", bindings)); - Assert.assertEquals(1L, evalLong("2.0 != 1.0", bindings)); - - Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings)); - Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings)); - Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings)); - Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings)); - - Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings)); - Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings)); - Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings)); - Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings)); - - Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings)); - Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings)); - Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings)); - Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings)); - - Assert.assertEquals(1L, evalLong("!-1.0", bindings)); - Assert.assertEquals(1L, evalLong("!0.0", bindings)); - Assert.assertEquals(0L, evalLong("!2.0", bindings)); - - assertEquals(3.5, evalDouble("2.0 + 1.5", bindings), 0.0001); - assertEquals(0.5, evalDouble("2.0 - 1.5", bindings), 0.0001); - assertEquals(3.0, evalDouble("2.0 * 1.5", bindings), 0.0001); - assertEquals(4.0, evalDouble("2.0 / 0.5", bindings), 0.0001); - assertEquals(0.2, evalDouble("2.0 % 0.3", bindings), 0.0001); - assertEquals(8.0, evalDouble("2.0 ^ 3.0", bindings), 0.0001); - assertEquals(-1.5, evalDouble("-1.5", bindings), 0.0001); + assertEquals(3.5, evalDouble("2.0 + 1.5", bindings), 0.0001); + assertEquals(0.5, evalDouble("2.0 - 1.5", bindings), 0.0001); + assertEquals(3.0, evalDouble("2.0 * 1.5", bindings), 0.0001); + assertEquals(4.0, evalDouble("2.0 / 0.5", bindings), 0.0001); + assertEquals(0.2, evalDouble("2.0 % 0.3", bindings), 0.0001); + assertEquals(8.0, evalDouble("2.0 ^ 3.0", bindings), 0.0001); + assertEquals(-1.5, evalDouble("-1.5", bindings), 0.0001); - assertEquals(2.0, evalDouble("sqrt(4.0)", bindings), 0.0001); - assertEquals(2.0, evalDouble("if(1.0, 2.0, 3.0)", bindings), 0.0001); - assertEquals(3.0, evalDouble("if(0.0, 2.0, 3.0)", bindings), 0.0001); - } - finally { - ExpressionProcessing.initializeForTests(); - } + assertEquals(2.0, evalDouble("sqrt(4.0)", bindings), 0.0001); + assertEquals(2.0, evalDouble("if(1.0, 2.0, 3.0)", bindings), 0.0001); + assertEquals(3.0, evalDouble("if(0.0, 2.0, 3.0)", bindings), 0.0001); } @Test @@ -934,56 +890,29 @@ public class EvalTest extends InitializedNullHandlingTest ImmutableMap.of("x", 100L, "y", 100L, "z", 100D, "w", 100D) ); - try { - ExpressionProcessing.initializeForStrictBooleansTests(false); - ExprEval eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings); - Assert.assertTrue(eval.asBoolean()); - assertEquals(ExpressionType.DOUBLE, eval.type()); + ExprEval eval = Parser.parse("x==y", ExprMacroTable.nil()).eval(bindings); + Assert.assertTrue(eval.asBoolean()); + assertEquals(ExpressionType.LONG, eval.type()); - eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings); - Assert.assertFalse(eval.asBoolean()); - assertEquals(ExpressionType.DOUBLE, eval.type()); + eval = Parser.parse("x!=y", ExprMacroTable.nil()).eval(bindings); + Assert.assertFalse(eval.asBoolean()); + assertEquals(ExpressionType.LONG, eval.type()); - eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings); - Assert.assertTrue(eval.asBoolean()); - assertEquals(ExpressionType.DOUBLE, eval.type()); + eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings); + Assert.assertTrue(eval.asBoolean()); + assertEquals(ExpressionType.LONG, eval.type()); - eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings); - Assert.assertFalse(eval.asBoolean()); - assertEquals(ExpressionType.DOUBLE, eval.type()); - } - finally { - ExpressionProcessing.initializeForTests(); - } - try { - ExpressionProcessing.initializeForStrictBooleansTests(true); - ExprEval eval = Parser.parse("x==y", ExprMacroTable.nil()).eval(bindings); - Assert.assertTrue(eval.asBoolean()); - assertEquals(ExpressionType.LONG, eval.type()); + eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings); + Assert.assertFalse(eval.asBoolean()); + assertEquals(ExpressionType.LONG, eval.type()); - eval = Parser.parse("x!=y", ExprMacroTable.nil()).eval(bindings); - Assert.assertFalse(eval.asBoolean()); - assertEquals(ExpressionType.LONG, eval.type()); + eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings); + Assert.assertTrue(eval.asBoolean()); + assertEquals(ExpressionType.LONG, eval.type()); - eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings); - Assert.assertTrue(eval.asBoolean()); - assertEquals(ExpressionType.LONG, eval.type()); - - eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings); - Assert.assertFalse(eval.asBoolean()); - assertEquals(ExpressionType.LONG, eval.type()); - - eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings); - Assert.assertTrue(eval.asBoolean()); - assertEquals(ExpressionType.LONG, eval.type()); - - eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings); - Assert.assertFalse(eval.asBoolean()); - assertEquals(ExpressionType.LONG, eval.type()); - } - finally { - ExpressionProcessing.initializeForTests(); - } + eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings); + Assert.assertFalse(eval.asBoolean()); + assertEquals(ExpressionType.LONG, eval.type()); } @Test @@ -991,142 +920,60 @@ public class EvalTest extends InitializedNullHandlingTest { Expr.ObjectBinding bindings = InputBindings.nilBindings(); - try { - ExpressionProcessing.initializeForStrictBooleansTests(true); - assertEquals(1L, eval("'true' && 'true'", bindings).value()); - assertEquals(0L, eval("'true' && 'false'", bindings).value()); - assertEquals(0L, eval("'false' && 'true'", bindings).value()); - assertEquals(0L, eval("'troo' && 'true'", bindings).value()); - assertEquals(0L, eval("'false' && 'false'", bindings).value()); + assertEquals(1L, eval("'true' && 'true'", bindings).value()); + assertEquals(0L, eval("'true' && 'false'", bindings).value()); + assertEquals(0L, eval("'false' && 'true'", bindings).value()); + assertEquals(0L, eval("'troo' && 'true'", bindings).value()); + assertEquals(0L, eval("'false' && 'false'", bindings).value()); - assertEquals(1L, eval("'true' || 'true'", bindings).value()); - assertEquals(1L, eval("'true' || 'false'", bindings).value()); - assertEquals(1L, eval("'false' || 'true'", bindings).value()); - assertEquals(1L, eval("'troo' || 'true'", bindings).value()); - assertEquals(0L, eval("'false' || 'false'", bindings).value()); + assertEquals(1L, eval("'true' || 'true'", bindings).value()); + assertEquals(1L, eval("'true' || 'false'", bindings).value()); + assertEquals(1L, eval("'false' || 'true'", bindings).value()); + assertEquals(1L, eval("'troo' || 'true'", bindings).value()); + assertEquals(0L, eval("'false' || 'false'", bindings).value()); - assertEquals(1L, eval("1 && 1", bindings).value()); - assertEquals(1L, eval("100 && 11", bindings).value()); - assertEquals(0L, eval("1 && 0", bindings).value()); - assertEquals(0L, eval("0 && 1", bindings).value()); - assertEquals(0L, eval("0 && 0", bindings).value()); + assertEquals(1L, eval("1 && 1", bindings).value()); + assertEquals(1L, eval("100 && 11", bindings).value()); + assertEquals(0L, eval("1 && 0", bindings).value()); + assertEquals(0L, eval("0 && 1", bindings).value()); + assertEquals(0L, eval("0 && 0", bindings).value()); - assertEquals(1L, eval("1 || 1", bindings).value()); - assertEquals(1L, eval("100 || 11", bindings).value()); - assertEquals(1L, eval("1 || 0", bindings).value()); - assertEquals(1L, eval("0 || 1", bindings).value()); - assertEquals(1L, eval("111 || 0", bindings).value()); - assertEquals(1L, eval("0 || 111", bindings).value()); - assertEquals(0L, eval("0 || 0", bindings).value()); + assertEquals(1L, eval("1 || 1", bindings).value()); + assertEquals(1L, eval("100 || 11", bindings).value()); + assertEquals(1L, eval("1 || 0", bindings).value()); + assertEquals(1L, eval("0 || 1", bindings).value()); + assertEquals(1L, eval("111 || 0", bindings).value()); + assertEquals(1L, eval("0 || 111", bindings).value()); + assertEquals(0L, eval("0 || 0", bindings).value()); - assertEquals(1L, eval("1.0 && 1.0", bindings).value()); - assertEquals(1L, eval("0.100 && 1.1", bindings).value()); - assertEquals(0L, eval("1.0 && 0.0", bindings).value()); - assertEquals(0L, eval("0.0 && 1.0", bindings).value()); - assertEquals(0L, eval("0.0 && 0.0", bindings).value()); + assertEquals(1L, eval("1.0 && 1.0", bindings).value()); + assertEquals(1L, eval("0.100 && 1.1", bindings).value()); + assertEquals(0L, eval("1.0 && 0.0", bindings).value()); + assertEquals(0L, eval("0.0 && 1.0", bindings).value()); + assertEquals(0L, eval("0.0 && 0.0", bindings).value()); - assertEquals(1L, eval("1.0 || 1.0", bindings).value()); - assertEquals(1L, eval("0.2 || 0.3", bindings).value()); - assertEquals(1L, eval("1.0 || 0.0", bindings).value()); - assertEquals(1L, eval("0.0 || 1.0", bindings).value()); - assertEquals(1L, eval("1.11 || 0.0", bindings).value()); - assertEquals(1L, eval("0.0 || 0.111", bindings).value()); - assertEquals(0L, eval("0.0 || 0.0", bindings).value()); + assertEquals(1L, eval("1.0 || 1.0", bindings).value()); + assertEquals(1L, eval("0.2 || 0.3", bindings).value()); + assertEquals(1L, eval("1.0 || 0.0", bindings).value()); + assertEquals(1L, eval("0.0 || 1.0", bindings).value()); + assertEquals(1L, eval("1.11 || 0.0", bindings).value()); + assertEquals(1L, eval("0.0 || 0.111", bindings).value()); + assertEquals(0L, eval("0.0 || 0.0", bindings).value()); - assertEquals(1L, eval("null || 1", bindings).value()); - assertEquals(1L, eval("1 || null", bindings).value()); - // in sql incompatible mode, null is false, so we return 0 - assertEquals(NullHandling.defaultLongValue(), eval("null || 0", bindings).valueOrDefault()); - assertEquals(NullHandling.defaultLongValue(), eval("0 || null", bindings).valueOrDefault()); - assertEquals(NullHandling.defaultLongValue(), eval("null || null", bindings).valueOrDefault()); + assertEquals(1L, eval("null || 1", bindings).value()); + assertEquals(1L, eval("1 || null", bindings).value()); + // in sql incompatible mode, null is false, so we return 0 + assertEquals(NullHandling.defaultLongValue(), eval("null || 0", bindings).valueOrDefault()); + assertEquals(NullHandling.defaultLongValue(), eval("0 || null", bindings).valueOrDefault()); + assertEquals(NullHandling.defaultLongValue(), eval("null || null", bindings).valueOrDefault()); - // in sql incompatible mode, null is false, so we return 0 - assertEquals(NullHandling.defaultLongValue(), eval("null && 1", bindings).valueOrDefault()); - assertEquals(NullHandling.defaultLongValue(), eval("1 && null", bindings).valueOrDefault()); - assertEquals(NullHandling.defaultLongValue(), eval("null && null", bindings).valueOrDefault()); - // if either side is false, output is false in both modes - assertEquals(0L, eval("null && 0", bindings).value()); - assertEquals(0L, eval("0 && null", bindings).value()); - } - finally { - // reset - ExpressionProcessing.initializeForTests(); - } - - try { - // turn on legacy insanity mode - ExpressionProcessing.initializeForStrictBooleansTests(false); - - assertEquals("true", eval("'true' && 'true'", bindings).value()); - assertEquals("false", eval("'true' && 'false'", bindings).value()); - assertEquals("false", eval("'false' && 'true'", bindings).value()); - assertEquals("troo", eval("'troo' && 'true'", bindings).value()); - assertEquals("false", eval("'false' && 'false'", bindings).value()); - - assertEquals("true", eval("'true' || 'true'", bindings).value()); - assertEquals("true", eval("'true' || 'false'", bindings).value()); - assertEquals("true", eval("'false' || 'true'", bindings).value()); - assertEquals("true", eval("'troo' || 'true'", bindings).value()); - assertEquals("false", eval("'false' || 'false'", bindings).value()); - - assertEquals(1.0, eval("1.0 && 1.0", bindings).value()); - assertEquals(1.1, eval("0.100 && 1.1", bindings).value()); - assertEquals(0.0, eval("1.0 && 0.0", bindings).value()); - assertEquals(0.0, eval("0.0 && 1.0", bindings).value()); - assertEquals(0.0, eval("0.0 && 0.0", bindings).value()); - - assertEquals(1.0, eval("1.0 || 1.0", bindings).value()); - assertEquals(0.2, eval("0.2 || 0.3", bindings).value()); - assertEquals(1.0, eval("1.0 || 0.0", bindings).value()); - assertEquals(1.0, eval("0.0 || 1.0", bindings).value()); - assertEquals(1.11, eval("1.11 || 0.0", bindings).value()); - assertEquals(0.111, eval("0.0 || 0.111", bindings).value()); - assertEquals(0.0, eval("0.0 || 0.0", bindings).value()); - - assertEquals(1L, eval("1 && 1", bindings).value()); - assertEquals(11L, eval("100 && 11", bindings).value()); - assertEquals(0L, eval("1 && 0", bindings).value()); - assertEquals(0L, eval("0 && 1", bindings).value()); - assertEquals(0L, eval("0 && 0", bindings).value()); - - assertEquals(1L, eval("1 || 1", bindings).value()); - assertEquals(100L, eval("100 || 11", bindings).value()); - assertEquals(1L, eval("1 || 0", bindings).value()); - assertEquals(1L, eval("0 || 1", bindings).value()); - assertEquals(111L, eval("111 || 0", bindings).value()); - assertEquals(111L, eval("0 || 111", bindings).value()); - assertEquals(0L, eval("0 || 0", bindings).value()); - - assertEquals(1.0, eval("1.0 && 1.0", bindings).value()); - assertEquals(1.1, eval("0.100 && 1.1", bindings).value()); - assertEquals(0.0, eval("1.0 && 0.0", bindings).value()); - assertEquals(0.0, eval("0.0 && 1.0", bindings).value()); - assertEquals(0.0, eval("0.0 && 0.0", bindings).value()); - - assertEquals(1.0, eval("1.0 || 1.0", bindings).value()); - assertEquals(0.2, eval("0.2 || 0.3", bindings).value()); - assertEquals(1.0, eval("1.0 || 0.0", bindings).value()); - assertEquals(1.0, eval("0.0 || 1.0", bindings).value()); - assertEquals(1.11, eval("1.11 || 0.0", bindings).value()); - assertEquals(0.111, eval("0.0 || 0.111", bindings).value()); - assertEquals(0.0, eval("0.0 || 0.0", bindings).value()); - - assertEquals(1L, eval("null || 1", bindings).value()); - assertEquals(1L, eval("1 || null", bindings).value()); - assertEquals(0L, eval("null || 0", bindings).value()); - Assert.assertNull(eval("0 || null", bindings).value()); - Assert.assertNull(eval("null || null", bindings).value()); - - Assert.assertNull(eval("null && 1", bindings).value()); - Assert.assertNull(eval("1 && null", bindings).value()); - Assert.assertNull(eval("null && 0", bindings).value()); - assertEquals(0L, eval("0 && null", bindings).value()); - assertNull(eval("null && null", bindings).value()); - } - finally { - // reset - ExpressionProcessing.initializeForTests(); - } + // in sql incompatible mode, null is false, so we return 0 + assertEquals(NullHandling.defaultLongValue(), eval("null && 1", bindings).valueOrDefault()); + assertEquals(NullHandling.defaultLongValue(), eval("1 && null", bindings).valueOrDefault()); + assertEquals(NullHandling.defaultLongValue(), eval("null && null", bindings).valueOrDefault()); + // if either side is false, output is false in both modes + assertEquals(0L, eval("null && 0", bindings).value()); + assertEquals(0L, eval("0 && null", bindings).value()); } @Test @@ -1143,86 +990,40 @@ public class EvalTest extends InitializedNullHandlingTest bindingsMap.put("b2", false); Expr.ObjectBinding bindings = InputBindings.forMap(bindingsMap); - try { - ExpressionProcessing.initializeForStrictBooleansTests(true); - assertEquals(1L, eval("s1 && s1", bindings).value()); - assertEquals(0L, eval("s1 && s2", bindings).value()); - assertEquals(0L, eval("s2 && s1", bindings).value()); - assertEquals(0L, eval("s2 && s2", bindings).value()); + assertEquals(1L, eval("s1 && s1", bindings).value()); + assertEquals(0L, eval("s1 && s2", bindings).value()); + assertEquals(0L, eval("s2 && s1", bindings).value()); + assertEquals(0L, eval("s2 && s2", bindings).value()); - assertEquals(1L, eval("s1 || s1", bindings).value()); - assertEquals(1L, eval("s1 || s2", bindings).value()); - assertEquals(1L, eval("s2 || s1", bindings).value()); - assertEquals(0L, eval("s2 || s2", bindings).value()); + assertEquals(1L, eval("s1 || s1", bindings).value()); + assertEquals(1L, eval("s1 || s2", bindings).value()); + assertEquals(1L, eval("s2 || s1", bindings).value()); + assertEquals(0L, eval("s2 || s2", bindings).value()); - assertEquals(1L, eval("l1 && l1", bindings).value()); - assertEquals(0L, eval("l1 && l2", bindings).value()); - assertEquals(0L, eval("l2 && l1", bindings).value()); - assertEquals(0L, eval("l2 && l2", bindings).value()); + assertEquals(1L, eval("l1 && l1", bindings).value()); + assertEquals(0L, eval("l1 && l2", bindings).value()); + assertEquals(0L, eval("l2 && l1", bindings).value()); + assertEquals(0L, eval("l2 && l2", bindings).value()); - assertEquals(1L, eval("b1 && b1", bindings).value()); - assertEquals(0L, eval("b1 && b2", bindings).value()); - assertEquals(0L, eval("b2 && b1", bindings).value()); - assertEquals(0L, eval("b2 && b2", bindings).value()); + assertEquals(1L, eval("b1 && b1", bindings).value()); + assertEquals(0L, eval("b1 && b2", bindings).value()); + assertEquals(0L, eval("b2 && b1", bindings).value()); + assertEquals(0L, eval("b2 && b2", bindings).value()); - assertEquals(1L, eval("d1 && d1", bindings).value()); - assertEquals(0L, eval("d1 && d2", bindings).value()); - assertEquals(0L, eval("d2 && d1", bindings).value()); - assertEquals(0L, eval("d2 && d2", bindings).value()); + assertEquals(1L, eval("d1 && d1", bindings).value()); + assertEquals(0L, eval("d1 && d2", bindings).value()); + assertEquals(0L, eval("d2 && d1", bindings).value()); + assertEquals(0L, eval("d2 && d2", bindings).value()); - assertEquals(1L, eval("b1", bindings).value()); - assertEquals(1L, eval("if(b1,1,0)", bindings).value()); - assertEquals(1L, eval("if(l1,1,0)", bindings).value()); - assertEquals(1L, eval("if(d1,1,0)", bindings).value()); - assertEquals(1L, eval("if(s1,1,0)", bindings).value()); - assertEquals(0L, eval("if(b2,1,0)", bindings).value()); - assertEquals(0L, eval("if(l2,1,0)", bindings).value()); - assertEquals(0L, eval("if(d2,1,0)", bindings).value()); - assertEquals(0L, eval("if(s2,1,0)", bindings).value()); - } - finally { - // reset - ExpressionProcessing.initializeForTests(); - } - - try { - // turn on legacy insanity mode - ExpressionProcessing.initializeForStrictBooleansTests(false); - - assertEquals("true", eval("s1 && s1", bindings).value()); - assertEquals("false", eval("s1 && s2", bindings).value()); - assertEquals("false", eval("s2 && s1", bindings).value()); - assertEquals("false", eval("s2 && s2", bindings).value()); - - assertEquals("true", eval("b1 && b1", bindings).value()); - assertEquals("false", eval("b1 && b2", bindings).value()); - assertEquals("false", eval("b2 && b1", bindings).value()); - assertEquals("false", eval("b2 && b2", bindings).value()); - - assertEquals(100L, eval("l1 && l1", bindings).value()); - assertEquals(0L, eval("l1 && l2", bindings).value()); - assertEquals(0L, eval("l2 && l1", bindings).value()); - assertEquals(0L, eval("l2 && l2", bindings).value()); - - assertEquals(1.1, eval("d1 && d1", bindings).value()); - assertEquals(0.0, eval("d1 && d2", bindings).value()); - assertEquals(0.0, eval("d2 && d1", bindings).value()); - assertEquals(0.0, eval("d2 && d2", bindings).value()); - - assertEquals("true", eval("b1", bindings).value()); - assertEquals(1L, eval("if(b1,1,0)", bindings).value()); - assertEquals(1L, eval("if(l1,1,0)", bindings).value()); - assertEquals(1L, eval("if(d1,1,0)", bindings).value()); - assertEquals(1L, eval("if(s1,1,0)", bindings).value()); - assertEquals(0L, eval("if(b2,1,0)", bindings).value()); - assertEquals(0L, eval("if(l2,1,0)", bindings).value()); - assertEquals(0L, eval("if(d2,1,0)", bindings).value()); - assertEquals(0L, eval("if(s2,1,0)", bindings).value()); - } - finally { - // reset - ExpressionProcessing.initializeForTests(); - } + assertEquals(1L, eval("b1", bindings).value()); + assertEquals(1L, eval("if(b1,1,0)", bindings).value()); + assertEquals(1L, eval("if(l1,1,0)", bindings).value()); + assertEquals(1L, eval("if(d1,1,0)", bindings).value()); + assertEquals(1L, eval("if(s1,1,0)", bindings).value()); + assertEquals(0L, eval("if(b2,1,0)", bindings).value()); + assertEquals(0L, eval("if(l2,1,0)", bindings).value()); + assertEquals(0L, eval("if(d2,1,0)", bindings).value()); + assertEquals(0L, eval("if(s2,1,0)", bindings).value()); } @Test @@ -1621,17 +1422,6 @@ public class EvalTest extends InitializedNullHandlingTest assertBestEffortOf(true, ExpressionType.LONG, 1L); assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L}); - try { - // in non-strict boolean mode, they are strings - ExpressionProcessing.initializeForStrictBooleansTests(false); - assertBestEffortOf(true, ExpressionType.STRING, "true"); - assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"}); - } - finally { - // reset - ExpressionProcessing.initializeForTests(); - } - // doubles assertBestEffortOf(1.0, ExpressionType.DOUBLE, 1.0); assertBestEffortOf(1.0f, ExpressionType.DOUBLE, 1.0); diff --git a/processing/src/test/java/org/apache/druid/math/expr/OutputTypeTest.java b/processing/src/test/java/org/apache/druid/math/expr/OutputTypeTest.java index 4ab7c7be0ed..088df4c9ef5 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/OutputTypeTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/OutputTypeTest.java @@ -70,29 +70,12 @@ public class OutputTypeTest extends InitializedNullHandlingTest assertOutputType("-y", inspector, ExpressionType.LONG); assertOutputType("-z", inspector, ExpressionType.DOUBLE); - try { - ExpressionProcessing.initializeForStrictBooleansTests(true); - assertOutputType("!'true'", inspector, ExpressionType.LONG); - assertOutputType("!1", inspector, ExpressionType.LONG); - assertOutputType("!x", inspector, ExpressionType.LONG); - assertOutputType("!y", inspector, ExpressionType.LONG); - assertOutputType("!1.1", inspector, ExpressionType.LONG); - assertOutputType("!z", inspector, ExpressionType.LONG); - } - finally { - // reset - ExpressionProcessing.initializeForTests(); - } - - try { - ExpressionProcessing.initializeForStrictBooleansTests(false); - assertOutputType("!1.1", inspector, ExpressionType.DOUBLE); - assertOutputType("!z", inspector, ExpressionType.DOUBLE); - } - finally { - // reset - ExpressionProcessing.initializeForTests(); - } + assertOutputType("!'true'", inspector, ExpressionType.LONG); + assertOutputType("!1", inspector, ExpressionType.LONG); + assertOutputType("!x", inspector, ExpressionType.LONG); + assertOutputType("!y", inspector, ExpressionType.LONG); + assertOutputType("!1.1", inspector, ExpressionType.LONG); + assertOutputType("!z", inspector, ExpressionType.LONG); } @Test @@ -126,61 +109,32 @@ public class OutputTypeTest extends InitializedNullHandlingTest assertOutputType("z^z_", inspector, ExpressionType.DOUBLE); assertOutputType("z%z_", inspector, ExpressionType.DOUBLE); - try { - ExpressionProcessing.initializeForStrictBooleansTests(true); - assertOutputType("y>y_", inspector, ExpressionType.LONG); - assertOutputType("y_=y", inspector, ExpressionType.LONG); - assertOutputType("y_==y", inspector, ExpressionType.LONG); - assertOutputType("y_!=y", inspector, ExpressionType.LONG); - assertOutputType("y_ && y", inspector, ExpressionType.LONG); - assertOutputType("y_ || y", inspector, ExpressionType.LONG); + assertOutputType("y>y_", inspector, ExpressionType.LONG); + assertOutputType("y_=y", inspector, ExpressionType.LONG); + assertOutputType("y_==y", inspector, ExpressionType.LONG); + assertOutputType("y_!=y", inspector, ExpressionType.LONG); + assertOutputType("y_ && y", inspector, ExpressionType.LONG); + assertOutputType("y_ || y", inspector, ExpressionType.LONG); - assertOutputType("z>y_", inspector, ExpressionType.LONG); - assertOutputType("z=z", inspector, ExpressionType.LONG); - assertOutputType("z==y", inspector, ExpressionType.LONG); - assertOutputType("z!=y", inspector, ExpressionType.LONG); - assertOutputType("z && y", inspector, ExpressionType.LONG); - assertOutputType("y || z", inspector, ExpressionType.LONG); + assertOutputType("z>y_", inspector, ExpressionType.LONG); + assertOutputType("z=z", inspector, ExpressionType.LONG); + assertOutputType("z==y", inspector, ExpressionType.LONG); + assertOutputType("z!=y", inspector, ExpressionType.LONG); + assertOutputType("z && y", inspector, ExpressionType.LONG); + assertOutputType("y || z", inspector, ExpressionType.LONG); - assertOutputType("z>z_", inspector, ExpressionType.LONG); - assertOutputType("z=z", inspector, ExpressionType.LONG); - assertOutputType("z==z_", inspector, ExpressionType.LONG); - assertOutputType("z!=z_", inspector, ExpressionType.LONG); - assertOutputType("z && z_", inspector, ExpressionType.LONG); - assertOutputType("z_ || z", inspector, ExpressionType.LONG); - } - finally { - ExpressionProcessing.initializeForTests(); - } - try { - ExpressionProcessing.initializeForStrictBooleansTests(false); - assertOutputType("z>y_", inspector, ExpressionType.DOUBLE); - assertOutputType("z=z", inspector, ExpressionType.DOUBLE); - assertOutputType("z==y", inspector, ExpressionType.DOUBLE); - assertOutputType("z!=y", inspector, ExpressionType.DOUBLE); - assertOutputType("z && y", inspector, ExpressionType.DOUBLE); - assertOutputType("y || z", inspector, ExpressionType.DOUBLE); - - assertOutputType("z>z_", inspector, ExpressionType.DOUBLE); - assertOutputType("z=z", inspector, ExpressionType.DOUBLE); - assertOutputType("z==z_", inspector, ExpressionType.DOUBLE); - assertOutputType("z!=z_", inspector, ExpressionType.DOUBLE); - assertOutputType("z && z_", inspector, ExpressionType.DOUBLE); - assertOutputType("z_ || z", inspector, ExpressionType.DOUBLE); - } - finally { - ExpressionProcessing.initializeForTests(); - } + assertOutputType("z>z_", inspector, ExpressionType.LONG); + assertOutputType("z=z", inspector, ExpressionType.LONG); + assertOutputType("z==z_", inspector, ExpressionType.LONG); + assertOutputType("z!=z_", inspector, ExpressionType.LONG); + assertOutputType("z && z_", inspector, ExpressionType.LONG); + assertOutputType("z_ || z", inspector, ExpressionType.LONG); assertOutputType("1*(2 + 3.0)", inspector, ExpressionType.DOUBLE); } diff --git a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java index b2c3d37d589..ba1c1c5b948 100644 --- a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java @@ -30,7 +30,6 @@ import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.Druids; import org.apache.druid.query.NestedDataTestUtils; import org.apache.druid.query.Query; @@ -668,76 +667,6 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString()); } - @Test - public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesNonStrictBooleans() throws Exception - { - - try { - ExpressionProcessing.initializeForStrictBooleansTests(false); - Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder() - .dataSource("test_datasource") - .intervals( - new MultipleIntervalSegmentSpec( - Collections.singletonList(Intervals.ETERNITY) - ) - ) - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(100) - .context(ImmutableMap.of()); - Query scanQuery = builder.build(); - final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")}; - List realtimeSegs = ImmutableList.of( - NestedDataTestUtils.createIncrementalIndex( - tempFolder, - NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2, - TestIndex.DEFAULT_JSON_INPUT_FORMAT, - NestedDataTestUtils.TIMESTAMP_SPEC, - NestedDataTestUtils.AUTO_DISCOVERY, - TransformSpec.NONE, - aggs, - Granularities.NONE, - true - ) - ); - List segs = NestedDataTestUtils.createSegments( - tempFolder, - closer, - NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2, - TestIndex.DEFAULT_JSON_INPUT_FORMAT, - NestedDataTestUtils.TIMESTAMP_SPEC, - NestedDataTestUtils.AUTO_DISCOVERY, - TransformSpec.NONE, - aggs, - Granularities.NONE, - true, - IndexSpec.DEFAULT - ); - - - final Sequence seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery); - final Sequence seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery); - - List resultsRealtime = seq.toList(); - List resultsSegments = seq2.toList(); - logResults(resultsSegments); - logResults(resultsRealtime); - Assert.assertEquals(1, resultsRealtime.size()); - Assert.assertEquals(resultsRealtime.size(), resultsSegments.size()); - Assert.assertEquals( - "[" - + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]", - resultsSegments.get(0).getEvents().toString() - ); - Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString()); - } - finally { - ExpressionProcessing.initializeForTests(); - } - } - @Test public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception { diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java deleted file mode 100644 index 8b60360b6c1..00000000000 --- a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.segment.filter; - -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import org.apache.druid.common.config.NullHandling; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.math.expr.ExpressionProcessing; -import org.apache.druid.query.filter.NotDimFilter; -import org.apache.druid.segment.CursorFactory; -import org.apache.druid.segment.IndexBuilder; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.Closeable; - -@RunWith(Parameterized.class) -public class ExpressionFilterNonStrictBooleansTest extends ExpressionFilterTest -{ - public ExpressionFilterNonStrictBooleansTest( - String testName, - IndexBuilder indexBuilder, - Function> finisher, - boolean cnf, - boolean optimize - ) - { - super(testName, indexBuilder, finisher, cnf, optimize); - } - - @Before - @Override - public void setup() - { - ExpressionProcessing.initializeForStrictBooleansTests(false); - } - - @Override - @Test - public void testComplement() - { - if (NullHandling.sqlCompatible()) { - assertFilterMatches(edf("dim5 == 'a'"), ImmutableList.of("0")); - // non-strict mode is wild - assertFilterMatches( - NotDimFilter.of(edf("dim5 == 'a'")), - ImmutableList.of("1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - assertFilterMatches( - edf("dim5 == ''"), ImmutableList.of("4") - ); - // non-strict mode! - assertFilterMatches( - NotDimFilter.of(edf("dim5 == ''")), ImmutableList.of("0", "1", "2", "3", "5", "6", "7", "8", "9") - ); - } else { - assertFilterMatches(edf("dim5 == 'a'"), ImmutableList.of("0")); - assertFilterMatches( - NotDimFilter.of(edf("dim5 == 'a'")), - ImmutableList.of("1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - } - } - - @Override - @Test - public void testMissingColumn() - { - if (NullHandling.replaceWithDefault()) { - assertFilterMatches( - edf("missing == ''"), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - assertFilterMatches( - edf("missing == otherMissing"), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - } else { - // AS per SQL standard null == null returns false. - assertFilterMatches(edf("missing == null"), ImmutableList.of()); - // in non-strict mode, madness happens - assertFilterMatches( - NotDimFilter.of(edf("missing == null")), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - // also this madness doesn't do madness - assertFilterMatches( - edf("missing == otherMissing"), - ImmutableList.of() - ); - assertFilterMatches( - NotDimFilter.of(edf("missing == otherMissing")), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - } - assertFilterMatches(edf("missing == '1'"), ImmutableList.of()); - assertFilterMatches(edf("missing == 2"), ImmutableList.of()); - if (NullHandling.replaceWithDefault()) { - // missing equivaluent to 0 - assertFilterMatches( - edf("missing < '2'"), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - assertFilterMatches( - edf("missing < 2"), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - assertFilterMatches( - edf("missing < 2.0"), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") - ); - } else { - // missing equivalent to null - assertFilterMatches(edf("missing < '2'"), ImmutableList.of()); - assertFilterMatches(edf("missing < 2"), ImmutableList.of()); - assertFilterMatches(edf("missing < 2.0"), ImmutableList.of()); - } - assertFilterMatches(edf("missing > '2'"), ImmutableList.of()); - assertFilterMatches(edf("missing > 2"), ImmutableList.of()); - assertFilterMatches(edf("missing > 2.0"), ImmutableList.of()); - assertFilterMatchesSkipVectorize(edf("like(missing, '1%')"), ImmutableList.of()); - } -} diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java index aa2d21ef9d3..9311a90b4e4 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java @@ -36,7 +36,6 @@ import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.filter.ExpressionDimFilter; import org.apache.druid.query.filter.Filter; @@ -46,10 +45,8 @@ import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -126,22 +123,10 @@ public class ExpressionFilterTest extends BaseFilterTest ); } - @Before - public void setup() - { - ExpressionProcessing.initializeForStrictBooleansTests(true); - } - - @After - public void teardown() - { - ExpressionProcessing.initializeForTests(); - } - @AfterClass public static void tearDown() throws Exception { - BaseFilterTest.tearDown(ColumnComparisonFilterTest.class.getName()); + BaseFilterTest.tearDown(ExpressionFilterTest.class.getName()); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/transform/TransformSpecTest.java b/processing/src/test/java/org/apache/druid/segment/transform/TransformSpecTest.java index 3ed0a51bf03..90413b6ab02 100644 --- a/processing/src/test/java/org/apache/druid/segment/transform/TransformSpecTest.java +++ b/processing/src/test/java/org/apache/druid/segment/transform/TransformSpecTest.java @@ -30,7 +30,6 @@ import org.apache.druid.data.input.impl.MapInputRowParser; import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.filter.AndDimFilter; import org.apache.druid.query.filter.SelectorDimFilter; @@ -208,68 +207,31 @@ public class TransformSpecTest extends InitializedNullHandlingTest @Test public void testBoolTransforms() { - try { - ExpressionProcessing.initializeForStrictBooleansTests(true); - final TransformSpec transformSpec = new TransformSpec( - null, - ImmutableList.of( - new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE), - new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE) - ) - ); + final TransformSpec transformSpec = new TransformSpec( + null, + ImmutableList.of( + new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE), + new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE) + ) + ); - Assert.assertEquals( - ImmutableSet.of("bool"), - transformSpec.getRequiredColumns() - ); + Assert.assertEquals( + ImmutableSet.of("bool"), + transformSpec.getRequiredColumns() + ); - final InputRowParser> parser = transformSpec.decorate(PARSER); - final InputRow row = parser.parseBatch(ROW1).get(0); + final InputRowParser> parser = transformSpec.decorate(PARSER); + final InputRow row = parser.parseBatch(ROW1).get(0); - Assert.assertNotNull(row); - Assert.assertEquals(1L, row.getRaw("truthy1")); - Assert.assertEquals(1L, row.getRaw("truthy2")); + Assert.assertNotNull(row); + Assert.assertEquals(1L, row.getRaw("truthy1")); + Assert.assertEquals(1L, row.getRaw("truthy2")); - final InputRow row2 = parser.parseBatch(ROW2).get(0); + final InputRow row2 = parser.parseBatch(ROW2).get(0); - Assert.assertNotNull(row2); - Assert.assertEquals(0L, row2.getRaw("truthy1")); - Assert.assertEquals(0L, row2.getRaw("truthy2")); - } - finally { - ExpressionProcessing.initializeForTests(); - } - try { - ExpressionProcessing.initializeForStrictBooleansTests(false); - final TransformSpec transformSpec = new TransformSpec( - null, - ImmutableList.of( - new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE), - new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE) - ) - ); - - Assert.assertEquals( - ImmutableSet.of("bool"), - transformSpec.getRequiredColumns() - ); - - final InputRowParser> parser = transformSpec.decorate(PARSER); - final InputRow row = parser.parseBatch(ROW1).get(0); - - Assert.assertNotNull(row); - Assert.assertEquals("true", row.getRaw("truthy1")); - Assert.assertEquals(1L, row.getRaw("truthy2")); - - final InputRow row2 = parser.parseBatch(ROW2).get(0); - - Assert.assertNotNull(row2); - Assert.assertEquals("false", row2.getRaw("truthy1")); - Assert.assertEquals(0L, row2.getRaw("truthy2")); - } - finally { - ExpressionProcessing.initializeForTests(); - } + Assert.assertNotNull(row2); + Assert.assertEquals(0L, row2.getRaw("truthy1")); + Assert.assertEquals(0L, row2.getRaw("truthy2")); } @Test diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRule.java index 97fa5b86a6a..c17e339e785 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRule.java @@ -27,7 +27,6 @@ import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rex.RexLiteral; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.InvalidSqlInput; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.InlineDataSource; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.planner.Calcites; @@ -122,7 +121,7 @@ public class DruidLogicalValuesRule extends RelOptRule } return ((Number) RexLiteral.value(literal)).longValue(); case BOOLEAN: - if (ExpressionProcessing.useStrictBooleans() && NullHandling.sqlCompatible() && literal.isNull()) { + if (NullHandling.sqlCompatible() && literal.isNull()) { return null; } return literal.isAlwaysTrue() ? 1L : 0L; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java index 5a6db426d29..f84c8455861 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java @@ -31,7 +31,6 @@ import org.apache.calcite.util.TimestampString; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidExceptionMatcher; import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.sql.calcite.planner.DruidTypeSystem; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.testing.InitializedNullHandlingTest; @@ -144,7 +143,7 @@ public class DruidLogicalValuesRuleTest { RexLiteral literal = REX_BUILDER.makeLiteral(null, REX_BUILDER.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN)); - if (NullHandling.sqlCompatible() && ExpressionProcessing.useStrictBooleans()) { + if (NullHandling.sqlCompatible()) { final Object fromLiteral = DruidLogicalValuesRule.getValueFromLiteral(literal, DEFAULT_CONTEXT); Assert.assertNull(fromLiteral); } else {