Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove druid.expressions.useStrictBooleans in favor of always being true (#17568)

This commit is contained in:
Clint Wylie 2024-12-17 18:49:16 -08:00 committed by GitHub
parent 8b81c91979
commit a44ab109d5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 206 additions and 1364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java
View File

@ -31,7 +31,6 @@ import org.apache.druid.data.input.impl.LongDimensionSchema;
import org.apache.druid.data.input.impl.StringDimensionSchema;
import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.segment.AutoTypeColumnSchema;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
@ -56,71 +55,6 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest
);
private InputSourceSampler inputSourceSampler = new InputSourceSampler(OBJECT_MAPPER);
@Test
public void testDiscoveredTypesNonStrictBooleans()
{
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
final SamplerResponse response = inputSourceSampler.sample(
inputSource,
new JsonInputFormat(null, null, null, null, null),
DataSchema.builder()
.withDataSource("test")
.withTimestamp(new TimestampSpec("t", null, null))
.withDimensions(DimensionsSpec.builder().useSchemaDiscovery(true).build())
.build(),
null
);
Assert.assertEquals(6, response.getNumRowsRead());
Assert.assertEquals(5, response.getNumRowsIndexed());
Assert.assertEquals(6, response.getData().size());
Assert.assertEquals(
ImmutableList.of(
new StringDimensionSchema("string"),
new LongDimensionSchema("long"),
new DoubleDimensionSchema("double"),
new StringDimensionSchema("bool"),
new StringDimensionSchema("variant"),
new AutoTypeColumnSchema("array", null),
new AutoTypeColumnSchema("nested", null)
),
response.getLogicalDimensions()
);
Assert.assertEquals(
ImmutableList.of(
new AutoTypeColumnSchema("string", null),
new AutoTypeColumnSchema("long", null),
new AutoTypeColumnSchema("double", null),
new AutoTypeColumnSchema("bool", null),
new AutoTypeColumnSchema("variant", null),
new AutoTypeColumnSchema("array", null),
new AutoTypeColumnSchema("nested", null)
),
response.getPhysicalDimensions()
);
Assert.assertEquals(
RowSignature.builder()
.addTimeColumn()
.add("string", ColumnType.STRING)
.add("long", ColumnType.LONG)
.add("double", ColumnType.DOUBLE)
.add("bool", ColumnType.STRING)
.add("variant", ColumnType.STRING)
.add("array", ColumnType.LONG_ARRAY)
.add("nested", ColumnType.NESTED_DATA)
.build(),
response.getLogicalSegmentSchema()
);
}
finally {
ExpressionProcessing.initializeForTests();
}
}
@Test
public void testDiscoveredTypesStrictBooleans()
{

4
processing/src/main/java/org/apache/druid/common/config/NullHandling.java
View File

@ -22,7 +22,6 @@ package org.apache.druid.common.config;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.inject.Inject;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.ValueMatcher;
@ -130,8 +129,7 @@ public class NullHandling
public static boolean useThreeValueLogic()
{
return sqlCompatible() &&
INSTANCE.isUseThreeValueLogicForNativeFilters() &&
ExpressionProcessing.useStrictBooleans();
INSTANCE.isUseThreeValueLogicForNativeFilters();
}
@Nullable

10
processing/src/main/java/org/apache/druid/math/expr/BinaryEvalOpExprBase.java
View File

@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableSet;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.column.Types;
import javax.annotation.Nullable;
import java.util.Objects;
@ -206,9 +205,6 @@ abstract class BinaryBooleanOpExprBase extends BinaryOpExprBase
result = evalDouble(leftVal.asDouble(), rightVal.asDouble());
break;
}
if (!ExpressionProcessing.useStrictBooleans() && !type.is(ExprType.STRING) && !type.isArray()) {
return ExprEval.ofBoolean(result, type);
}
return ExprEval.ofLongBoolean(result);
}
@ -224,11 +220,7 @@ abstract class BinaryBooleanOpExprBase extends BinaryOpExprBase
@Override
public ExpressionType getOutputType(InputBindingInspector inspector)
{
ExpressionType implicitCast = super.getOutputType(inspector);
if (ExpressionProcessing.useStrictBooleans() || Types.isNullOr(implicitCast, ExprType.STRING)) {
return ExpressionType.LONG;
}
return implicitCast;
return ExpressionType.LONG;
}
@Override

20
processing/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java
View File

@ -340,9 +340,6 @@ class BinAndExpr extends BinaryOpExprBase
public ExprEval eval(ObjectBinding bindings)
{
ExprEval leftVal = left.eval(bindings);
if (!ExpressionProcessing.useStrictBooleans()) {
return leftVal.asBoolean() ? right.eval(bindings) : leftVal;
}
// if left is false, always false
if (leftVal.value() != null && !leftVal.asBoolean()) {
@ -376,9 +373,7 @@ class BinAndExpr extends BinaryOpExprBase
@Override
public boolean canVectorize(InputBindingInspector inspector)
{
return ExpressionProcessing.useStrictBooleans() &&
inspector.areSameTypes(left, right) &&
inspector.canVectorize(left, right);
return inspector.areSameTypes(left, right) && inspector.canVectorize(left, right);
}
@Override
@ -391,9 +386,6 @@ class BinAndExpr extends BinaryOpExprBase
@Override
public ExpressionType getOutputType(InputBindingInspector inspector)
{
if (!ExpressionProcessing.useStrictBooleans()) {
return super.getOutputType(inspector);
}
return ExpressionType.LONG;
}
}
@ -415,9 +407,6 @@ class BinOrExpr extends BinaryOpExprBase
public ExprEval eval(ObjectBinding bindings)
{
ExprEval leftVal = left.eval(bindings);
if (!ExpressionProcessing.useStrictBooleans()) {
return leftVal.asBoolean() ? leftVal : right.eval(bindings);
}
// if left is true, always true
if (leftVal.value() != null && leftVal.asBoolean()) {
@ -454,9 +443,7 @@ class BinOrExpr extends BinaryOpExprBase
public boolean canVectorize(InputBindingInspector inspector)
{
return ExpressionProcessing.useStrictBooleans() &&
inspector.areSameTypes(left, right) &&
inspector.canVectorize(left, right);
return inspector.areSameTypes(left, right) && inspector.canVectorize(left, right);
}
@Override
@ -469,9 +456,6 @@ class BinOrExpr extends BinaryOpExprBase
@Override
public ExpressionType getOutputType(InputBindingInspector inspector)
{
if (!ExpressionProcessing.useStrictBooleans()) {
return super.getOutputType(inspector);
}
return ExpressionType.LONG;
}
}

33
processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
View File

@ -242,11 +242,7 @@ public abstract class ExprEval<T>
if (Number.class.isAssignableFrom(next) || next == String.class || next == Boolean.class) {
// coerce booleans
if (next == Boolean.class) {
if (ExpressionProcessing.useStrictBooleans()) {
next = Long.class;
} else {
next = String.class;
}
next = Long.class;
}
if (existing == null) {
return next;
@ -350,28 +346,6 @@ public abstract class ExprEval<T>
return new ArrayExprEval(outputType, value);
}
/**
* Convert a boolean back into native expression type
*
* Do not use this method unless {@link ExpressionProcessing#useStrictBooleans()} is set to false.
* {@link ExpressionType#LONG} is the Druid boolean unless this mode is enabled, so use {@link #ofLongBoolean}
* instead.
*/
@Deprecated
public static ExprEval ofBoolean(boolean value, ExpressionType type)
{
switch (type.getType()) {
case DOUBLE:
return of(Evals.asDouble(value));
case LONG:
return ofLongBoolean(value);
case STRING:
return of(String.valueOf(value));
default:
throw new Types.InvalidCastBooleanException(type);
}
}
/**
* Convert a boolean into a long expression type
*/
@ -421,10 +395,7 @@ public abstract class ExprEval<T>
return new LongExprEval((Number) val);
}
if (val instanceof Boolean) {
if (ExpressionProcessing.useStrictBooleans()) {
return ofLongBoolean((Boolean) val);
}
return new StringExprEval(String.valueOf(val));
return ofLongBoolean((Boolean) val);
}
if (val instanceof Long[]) {
final Long[] inputArray = (Long[]) val;

15
processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java
View File

@ -48,12 +48,6 @@ public class ExpressionProcessing
INSTANCE = new ExpressionProcessingConfig(null, null, null, null);
}
@VisibleForTesting
public static void initializeForStrictBooleansTests(boolean useStrict)
{
INSTANCE = new ExpressionProcessingConfig(useStrict, null, null, null);
}
@VisibleForTesting
public static void initializeForHomogenizeNullMultiValueStrings()
{
@ -66,15 +60,6 @@ public class ExpressionProcessing
INSTANCE = new ExpressionProcessingConfig(null, null, null, true);
}
/**
* All boolean expressions are {@link ExpressionType#LONG}
*/
public static boolean useStrictBooleans()
{
checkInitialized();
return INSTANCE.isUseStrictBooleans();
}
/**
* All {@link ExprType#ARRAY} values will be converted to {@link ExpressionType#STRING} by their column selectors
* (not within expression processing) to be treated as multi-value strings instead of native arrays.

17
processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
View File

@ -30,6 +30,7 @@ public class ExpressionProcessingConfig
{
private static final Logger LOG = new Logger(ExpressionProcessingConfig.class);
@Deprecated
public static final String NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING = "druid.expressions.useStrictBooleans";
// Coerce arrays to multi value strings
public static final String PROCESS_ARRAYS_AS_MULTIVALUE_STRINGS_CONFIG_STRING =
@ -39,9 +40,6 @@ public class ExpressionProcessingConfig
"druid.expressions.homogenizeNullMultiValueStringArrays";
public static final String ALLOW_VECTORIZE_FALLBACK = "druid.expressions.allowVectorizeFallback";
@JsonProperty("useStrictBooleans")
private final boolean useStrictBooleans;
@JsonProperty("processArraysAsMultiValueStrings")
private final boolean processArraysAsMultiValueStrings;
@ -51,9 +49,13 @@ public class ExpressionProcessingConfig
@JsonProperty("allowVectorizeFallback")
private final boolean allowVectorizeFallback;
@Deprecated
@JsonProperty("useStrictBooleans")
private final boolean useStrictBooleans;
@JsonCreator
public ExpressionProcessingConfig(
@JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans,
@Deprecated @JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans,
@JsonProperty("processArraysAsMultiValueStrings") @Nullable Boolean processArraysAsMultiValueStrings,
@JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays,
@JsonProperty("allowVectorizeFallback") @Nullable Boolean allowVectorizeFallback
@ -83,17 +85,12 @@ public class ExpressionProcessingConfig
final String docsBaseFormat = "https://druid.apache.org/docs/%s/querying/sql-data-types#%s";
if (!this.useStrictBooleans) {
LOG.warn(
"druid.expressions.useStrictBooleans set to 'false', we recommend using 'true' if using SQL to query Druid for the most SQL compliant behavior, see %s for details",
"druid.expressions.useStrictBooleans set to 'false', but has been removed from Druid and is always 'true' now for the most SQL compliant behavior, see %s for details",
StringUtils.format(docsBaseFormat, version, "boolean-logic")
);
}
}
public boolean isUseStrictBooleans()
{
return useStrictBooleans;
}
public boolean processArraysAsMultiValueStrings()
{
return processArraysAsMultiValueStrings;

13
processing/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java
View File

@ -26,7 +26,6 @@ import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.vector.ExprVectorProcessor;
import org.apache.druid.math.expr.vector.VectorMathProcessors;
import org.apache.druid.math.expr.vector.VectorProcessors;
import org.apache.druid.segment.column.Types;
import javax.annotation.Nullable;
import java.math.BigInteger;
@ -181,11 +180,6 @@ class UnaryNotExpr extends UnaryExpr
if (NullHandling.sqlCompatible() && (ret.value() == null)) {
return ExprEval.of(null);
}
if (!ExpressionProcessing.useStrictBooleans()) {
// conforming to other boolean-returning binary operators
ExpressionType retType = ret.type().is(ExprType.DOUBLE) ? ExpressionType.DOUBLE : ExpressionType.LONG;
return ExprEval.ofBoolean(!ret.asBoolean(), retType);
}
return ExprEval.ofLongBoolean(!ret.asBoolean());
}
@ -193,13 +187,6 @@ class UnaryNotExpr extends UnaryExpr
@Override
public ExpressionType getOutputType(InputBindingInspector inspector)
{
if (!ExpressionProcessing.useStrictBooleans()) {
ExpressionType implicitCast = super.getOutputType(inspector);
if (Types.is(implicitCast, ExprType.STRING)) {
return ExpressionType.LONG;
}
return implicitCast;
}
return ExpressionType.LONG;
}

467
processing/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java
View File

@ -23,7 +23,6 @@ import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprType;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.segment.column.Types;
@ -33,50 +32,6 @@ import java.util.function.Supplier;
public class VectorComparisonProcessors
{
@Deprecated
public static <T> ExprVectorProcessor<T> makeComparisonProcessor(
Expr.VectorInputBindingInspector inspector,
Expr left,
Expr right,
Supplier<LongOutObjectsInFunctionVectorProcessor> longOutStringsInFunctionVectorProcessor,
Supplier<LongOutLongsInFunctionVectorValueProcessor> longOutLongsInProcessor,
Supplier<DoubleOutLongDoubleInFunctionVectorValueProcessor> doubleOutLongDoubleInProcessor,
Supplier<DoubleOutDoubleLongInFunctionVectorValueProcessor> doubleOutDoubleLongInProcessor,
Supplier<DoubleOutDoublesInFunctionVectorValueProcessor> doubleOutDoublesInProcessor
)
{
assert !ExpressionProcessing.useStrictBooleans();
final ExpressionType leftType = left.getOutputType(inspector);
final ExpressionType rightType = right.getOutputType(inspector);
ExprVectorProcessor<?> processor = null;
if (Types.is(leftType, ExprType.STRING)) {
if (Types.isNullOr(rightType, ExprType.STRING)) {
processor = longOutStringsInFunctionVectorProcessor.get();
} else {
processor = doubleOutDoublesInProcessor.get();
}
} else if (leftType == null) {
if (Types.isNullOr(rightType, ExprType.STRING)) {
processor = longOutStringsInFunctionVectorProcessor.get();
}
} else if (leftType.is(ExprType.DOUBLE) || Types.is(rightType, ExprType.DOUBLE)) {
processor = doubleOutDoublesInProcessor.get();
}
if (processor != null) {
return (ExprVectorProcessor<T>) processor;
}
// fall through to normal math processor logic
return VectorMathProcessors.makeMathProcessor(
inspector,
left,
right,
longOutLongsInProcessor,
doubleOutLongDoubleInProcessor,
doubleOutDoubleLongInProcessor,
doubleOutDoublesInProcessor
);
}
public static <T> ExprVectorProcessor<T> makeBooleanProcessor(
Expr.VectorInputBindingInspector inspector,
Expr left,
@ -131,75 +86,6 @@ public class VectorComparisonProcessors
Expr right
)
{
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Objects.equals(leftVal, rightVal));
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left == right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(left == right);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(left == right);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(left == right);
}
}
);
}
return makeBooleanProcessor(
inspector,
left,
@ -275,75 +161,6 @@ public class VectorComparisonProcessors
Expr right
)
{
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(!Objects.equals(leftVal, rightVal));
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left != right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(left != right);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(left != right);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(left != right);
}
}
);
}
return makeBooleanProcessor(
inspector,
left,
@ -419,77 +236,6 @@ public class VectorComparisonProcessors
Expr right
)
{
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) >= 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left >= right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(Double.compare(left, right) >= 0);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(Double.compare(left, right) >= 0);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(Double.compare(left, right) >= 0);
}
}
);
}
return makeBooleanProcessor(
inspector,
left,
@ -567,77 +313,6 @@ public class VectorComparisonProcessors
Expr right
)
{
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) > 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left > right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(Double.compare(left, right) > 0);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(Double.compare(left, right) > 0);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(Double.compare(left, right) > 0);
}
}
);
}
return makeBooleanProcessor(
inspector,
left,
@ -715,77 +390,6 @@ public class VectorComparisonProcessors
Expr right
)
{
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) <= 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left <= right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(Double.compare(left, right) <= 0);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(Double.compare(left, right) <= 0);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(Double.compare(left, right) <= 0);
}
}
);
}
return makeBooleanProcessor(
inspector,
left,
@ -863,77 +467,6 @@ public class VectorComparisonProcessors
Expr right
)
{
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) < 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left < right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(Double.compare(left, right) < 0);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(Double.compare(left, right) < 0);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(Double.compare(left, right) < 0);
}
}
);
}
return makeBooleanProcessor(
inspector,
left,

26
processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java
View File

@ -25,7 +25,6 @@ import org.apache.druid.error.DruidException;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprType;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Exprs;
import org.apache.druid.segment.column.Types;
@ -658,25 +657,14 @@ public class VectorProcessors
}
};
} else if (Types.is(inputType, ExprType.DOUBLE)) {
if (!ExpressionProcessing.useStrictBooleans()) {
processor = new DoubleOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize)
processor = new LongOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize)
{
@Override
public long apply(double input)
{
@Override
public double apply(double input)
{
return Evals.asDouble(!Evals.asBoolean(input));
}
};
} else {
processor = new LongOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize)
{
@Override
public long apply(double input)
{
return Evals.asLong(!Evals.asBoolean(input));
}
};
}
return Evals.asLong(!Evals.asBoolean(input));
}
};
}
if (processor == null) {
throw Exprs.cannotVectorize();

10
processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java
View File

@ -21,7 +21,6 @@ package org.apache.druid.segment.filter;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter;
@ -45,11 +44,10 @@ import java.util.Set;
/**
* Nice filter you have there... NOT!
*
* If {@link ExpressionProcessing#useStrictBooleans()} and {@link NullHandling#sqlCompatible()} are both true, this
* filter inverts the {@code includeUnknown} flag to properly map Druids native two-valued logic (true, false) to SQL
* three-valued logic (true, false, unknown). At the top level, this flag is always passed in as 'false', and is only
* flipped by this filter. Other logical filters ({@link AndFilter} and {@link OrFilter}) propagate the value of
* {@code includeUnknown} to their children.
* If {@link NullHandling#sqlCompatible()} is true, this filter inverts the {@code includeUnknown} flag to properly
* map Druids native two-valued logic (true, false) to SQL three-valued logic (true, false, unknown). At the top level,
* this flag is always passed in as 'false', and is only flipped by this filter. Other logical filters
* ({@link AndFilter} and {@link OrFilter}) propagate the value of {@code includeUnknown} to their children.
*
* For example, if the base filter is equality, by default value matchers and indexes only return true for the rows
* that are equal to the value. When wrapped in a not filter, the not filter indicates that the equality matchers and

468
processing/src/test/java/org/apache/druid/math/expr/EvalTest.java
View File

@ -41,7 +41,6 @@ import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
/**
*/
@ -84,93 +83,50 @@ public class EvalTest extends InitializedNullHandlingTest
assertEquals(2.0, evalDouble("\"x\"", bindings), 0.0001);
assertEquals(304.0, evalDouble("300 + \"x\" * 2", bindings), 0.0001);
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
Assert.assertFalse(evalDouble("1.0 && 0.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("1.0 && 2.0", bindings) > 0.0);
Assert.assertEquals(0L, evalLong("1.0 && 0.0", bindings));
Assert.assertEquals(1L, evalLong("1.0 && 2.0", bindings));
Assert.assertTrue(evalDouble("1.0 || 0.0", bindings) > 0.0);
Assert.assertFalse(evalDouble("0.0 || 0.0", bindings) > 0.0);
Assert.assertEquals(1L, evalLong("1.0 || 0.0", bindings));
Assert.assertEquals(0L, evalLong("0.0 || 0.0", bindings));
Assert.assertTrue(evalDouble("2.0 > 1.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("2.0 >= 2.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("1.0 < 2.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("2.0 <= 2.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("2.0 == 2.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("2.0 != 1.0", bindings) > 0.0);
Assert.assertEquals(1L, evalLong("2.0 > 1.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 >= 2.0", bindings));
Assert.assertEquals(1L, evalLong("1.0 < 2.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 <= 2.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 == 2.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 != 1.0", bindings));
Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings));
Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings));
Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings));
Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings));
Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings));
Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings));
Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings));
Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings));
Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings));
Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings));
Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings));
Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings));
Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings));
Assert.assertTrue(evalDouble("!-1.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("!0.0", bindings) > 0.0);
Assert.assertFalse(evalDouble("!2.0", bindings) > 0.0);
}
finally {
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
Assert.assertEquals(0L, evalLong("1.0 && 0.0", bindings));
Assert.assertEquals(1L, evalLong("1.0 && 2.0", bindings));
Assert.assertEquals(1L, evalLong("!-1.0", bindings));
Assert.assertEquals(1L, evalLong("!0.0", bindings));
Assert.assertEquals(0L, evalLong("!2.0", bindings));
Assert.assertEquals(1L, evalLong("1.0 || 0.0", bindings));
Assert.assertEquals(0L, evalLong("0.0 || 0.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 > 1.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 >= 2.0", bindings));
Assert.assertEquals(1L, evalLong("1.0 < 2.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 <= 2.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 == 2.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 != 1.0", bindings));
Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings));
Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings));
Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings));
Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings));
Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings));
Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings));
Assert.assertEquals(1L, evalLong("!-1.0", bindings));
Assert.assertEquals(1L, evalLong("!0.0", bindings));
Assert.assertEquals(0L, evalLong("!2.0", bindings));
assertEquals(3.5, evalDouble("2.0 + 1.5", bindings), 0.0001);
assertEquals(0.5, evalDouble("2.0 - 1.5", bindings), 0.0001);
assertEquals(3.0, evalDouble("2.0 * 1.5", bindings), 0.0001);
assertEquals(4.0, evalDouble("2.0 / 0.5", bindings), 0.0001);
assertEquals(0.2, evalDouble("2.0 % 0.3", bindings), 0.0001);
assertEquals(8.0, evalDouble("2.0 ^ 3.0", bindings), 0.0001);
assertEquals(-1.5, evalDouble("-1.5", bindings), 0.0001);
assertEquals(3.5, evalDouble("2.0 + 1.5", bindings), 0.0001);
assertEquals(0.5, evalDouble("2.0 - 1.5", bindings), 0.0001);
assertEquals(3.0, evalDouble("2.0 * 1.5", bindings), 0.0001);
assertEquals(4.0, evalDouble("2.0 / 0.5", bindings), 0.0001);
assertEquals(0.2, evalDouble("2.0 % 0.3", bindings), 0.0001);
assertEquals(8.0, evalDouble("2.0 ^ 3.0", bindings), 0.0001);
assertEquals(-1.5, evalDouble("-1.5", bindings), 0.0001);
assertEquals(2.0, evalDouble("sqrt(4.0)", bindings), 0.0001);
assertEquals(2.0, evalDouble("if(1.0, 2.0, 3.0)", bindings), 0.0001);
assertEquals(3.0, evalDouble("if(0.0, 2.0, 3.0)", bindings), 0.0001);
}
finally {
ExpressionProcessing.initializeForTests();
}
assertEquals(2.0, evalDouble("sqrt(4.0)", bindings), 0.0001);
assertEquals(2.0, evalDouble("if(1.0, 2.0, 3.0)", bindings), 0.0001);
assertEquals(3.0, evalDouble("if(0.0, 2.0, 3.0)", bindings), 0.0001);
}
@Test
@ -934,56 +890,29 @@ public class EvalTest extends InitializedNullHandlingTest
ImmutableMap.of("x", 100L, "y", 100L, "z", 100D, "w", 100D)
);
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
ExprEval eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.DOUBLE, eval.type());
ExprEval eval = Parser.parse("x==y", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.DOUBLE, eval.type());
eval = Parser.parse("x!=y", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.DOUBLE, eval.type());
eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.DOUBLE, eval.type());
}
finally {
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
ExprEval eval = Parser.parse("x==y", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("x!=y", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
}
finally {
ExpressionProcessing.initializeForTests();
}
eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
}
@Test
@ -991,142 +920,60 @@ public class EvalTest extends InitializedNullHandlingTest
{
Expr.ObjectBinding bindings = InputBindings.nilBindings();
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
assertEquals(1L, eval("'true' && 'true'", bindings).value());
assertEquals(0L, eval("'true' && 'false'", bindings).value());
assertEquals(0L, eval("'false' && 'true'", bindings).value());
assertEquals(0L, eval("'troo' && 'true'", bindings).value());
assertEquals(0L, eval("'false' && 'false'", bindings).value());
assertEquals(1L, eval("'true' && 'true'", bindings).value());
assertEquals(0L, eval("'true' && 'false'", bindings).value());
assertEquals(0L, eval("'false' && 'true'", bindings).value());
assertEquals(0L, eval("'troo' && 'true'", bindings).value());
assertEquals(0L, eval("'false' && 'false'", bindings).value());
assertEquals(1L, eval("'true' || 'true'", bindings).value());
assertEquals(1L, eval("'true' || 'false'", bindings).value());
assertEquals(1L, eval("'false' || 'true'", bindings).value());
assertEquals(1L, eval("'troo' || 'true'", bindings).value());
assertEquals(0L, eval("'false' || 'false'", bindings).value());
assertEquals(1L, eval("'true' || 'true'", bindings).value());
assertEquals(1L, eval("'true' || 'false'", bindings).value());
assertEquals(1L, eval("'false' || 'true'", bindings).value());
assertEquals(1L, eval("'troo' || 'true'", bindings).value());
assertEquals(0L, eval("'false' || 'false'", bindings).value());
assertEquals(1L, eval("1 && 1", bindings).value());
assertEquals(1L, eval("100 && 11", bindings).value());
assertEquals(0L, eval("1 && 0", bindings).value());
assertEquals(0L, eval("0 && 1", bindings).value());
assertEquals(0L, eval("0 && 0", bindings).value());
assertEquals(1L, eval("1 && 1", bindings).value());
assertEquals(1L, eval("100 && 11", bindings).value());
assertEquals(0L, eval("1 && 0", bindings).value());
assertEquals(0L, eval("0 && 1", bindings).value());
assertEquals(0L, eval("0 && 0", bindings).value());
assertEquals(1L, eval("1 || 1", bindings).value());
assertEquals(1L, eval("100 || 11", bindings).value());
assertEquals(1L, eval("1 || 0", bindings).value());
assertEquals(1L, eval("0 || 1", bindings).value());
assertEquals(1L, eval("111 || 0", bindings).value());
assertEquals(1L, eval("0 || 111", bindings).value());
assertEquals(0L, eval("0 || 0", bindings).value());
assertEquals(1L, eval("1 || 1", bindings).value());
assertEquals(1L, eval("100 || 11", bindings).value());
assertEquals(1L, eval("1 || 0", bindings).value());
assertEquals(1L, eval("0 || 1", bindings).value());
assertEquals(1L, eval("111 || 0", bindings).value());
assertEquals(1L, eval("0 || 111", bindings).value());
assertEquals(0L, eval("0 || 0", bindings).value());
assertEquals(1L, eval("1.0 && 1.0", bindings).value());
assertEquals(1L, eval("0.100 && 1.1", bindings).value());
assertEquals(0L, eval("1.0 && 0.0", bindings).value());
assertEquals(0L, eval("0.0 && 1.0", bindings).value());
assertEquals(0L, eval("0.0 && 0.0", bindings).value());
assertEquals(1L, eval("1.0 && 1.0", bindings).value());
assertEquals(1L, eval("0.100 && 1.1", bindings).value());
assertEquals(0L, eval("1.0 && 0.0", bindings).value());
assertEquals(0L, eval("0.0 && 1.0", bindings).value());
assertEquals(0L, eval("0.0 && 0.0", bindings).value());
assertEquals(1L, eval("1.0 || 1.0", bindings).value());
assertEquals(1L, eval("0.2 || 0.3", bindings).value());
assertEquals(1L, eval("1.0 || 0.0", bindings).value());
assertEquals(1L, eval("0.0 || 1.0", bindings).value());
assertEquals(1L, eval("1.11 || 0.0", bindings).value());
assertEquals(1L, eval("0.0 || 0.111", bindings).value());
assertEquals(0L, eval("0.0 || 0.0", bindings).value());
assertEquals(1L, eval("1.0 || 1.0", bindings).value());
assertEquals(1L, eval("0.2 || 0.3", bindings).value());
assertEquals(1L, eval("1.0 || 0.0", bindings).value());
assertEquals(1L, eval("0.0 || 1.0", bindings).value());
assertEquals(1L, eval("1.11 || 0.0", bindings).value());
assertEquals(1L, eval("0.0 || 0.111", bindings).value());
assertEquals(0L, eval("0.0 || 0.0", bindings).value());
assertEquals(1L, eval("null || 1", bindings).value());
assertEquals(1L, eval("1 || null", bindings).value());
// in sql incompatible mode, null is false, so we return 0
assertEquals(NullHandling.defaultLongValue(), eval("null || 0", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("0 || null", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("null || null", bindings).valueOrDefault());
assertEquals(1L, eval("null || 1", bindings).value());
assertEquals(1L, eval("1 || null", bindings).value());
// in sql incompatible mode, null is false, so we return 0
assertEquals(NullHandling.defaultLongValue(), eval("null || 0", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("0 || null", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("null || null", bindings).valueOrDefault());
// in sql incompatible mode, null is false, so we return 0
assertEquals(NullHandling.defaultLongValue(), eval("null && 1", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("1 && null", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("null && null", bindings).valueOrDefault());
// if either side is false, output is false in both modes
assertEquals(0L, eval("null && 0", bindings).value());
assertEquals(0L, eval("0 && null", bindings).value());
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
try {
// turn on legacy insanity mode
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertEquals("true", eval("'true' && 'true'", bindings).value());
assertEquals("false", eval("'true' && 'false'", bindings).value());
assertEquals("false", eval("'false' && 'true'", bindings).value());
assertEquals("troo", eval("'troo' && 'true'", bindings).value());
assertEquals("false", eval("'false' && 'false'", bindings).value());
assertEquals("true", eval("'true' || 'true'", bindings).value());
assertEquals("true", eval("'true' || 'false'", bindings).value());
assertEquals("true", eval("'false' || 'true'", bindings).value());
assertEquals("true", eval("'troo' || 'true'", bindings).value());
assertEquals("false", eval("'false' || 'false'", bindings).value());
assertEquals(1.0, eval("1.0 && 1.0", bindings).value());
assertEquals(1.1, eval("0.100 && 1.1", bindings).value());
assertEquals(0.0, eval("1.0 && 0.0", bindings).value());
assertEquals(0.0, eval("0.0 && 1.0", bindings).value());
assertEquals(0.0, eval("0.0 && 0.0", bindings).value());
assertEquals(1.0, eval("1.0 || 1.0", bindings).value());
assertEquals(0.2, eval("0.2 || 0.3", bindings).value());
assertEquals(1.0, eval("1.0 || 0.0", bindings).value());
assertEquals(1.0, eval("0.0 || 1.0", bindings).value());
assertEquals(1.11, eval("1.11 || 0.0", bindings).value());
assertEquals(0.111, eval("0.0 || 0.111", bindings).value());
assertEquals(0.0, eval("0.0 || 0.0", bindings).value());
assertEquals(1L, eval("1 && 1", bindings).value());
assertEquals(11L, eval("100 && 11", bindings).value());
assertEquals(0L, eval("1 && 0", bindings).value());
assertEquals(0L, eval("0 && 1", bindings).value());
assertEquals(0L, eval("0 && 0", bindings).value());
assertEquals(1L, eval("1 || 1", bindings).value());
assertEquals(100L, eval("100 || 11", bindings).value());
assertEquals(1L, eval("1 || 0", bindings).value());
assertEquals(1L, eval("0 || 1", bindings).value());
assertEquals(111L, eval("111 || 0", bindings).value());
assertEquals(111L, eval("0 || 111", bindings).value());
assertEquals(0L, eval("0 || 0", bindings).value());
assertEquals(1.0, eval("1.0 && 1.0", bindings).value());
assertEquals(1.1, eval("0.100 && 1.1", bindings).value());
assertEquals(0.0, eval("1.0 && 0.0", bindings).value());
assertEquals(0.0, eval("0.0 && 1.0", bindings).value());
assertEquals(0.0, eval("0.0 && 0.0", bindings).value());
assertEquals(1.0, eval("1.0 || 1.0", bindings).value());
assertEquals(0.2, eval("0.2 || 0.3", bindings).value());
assertEquals(1.0, eval("1.0 || 0.0", bindings).value());
assertEquals(1.0, eval("0.0 || 1.0", bindings).value());
assertEquals(1.11, eval("1.11 || 0.0", bindings).value());
assertEquals(0.111, eval("0.0 || 0.111", bindings).value());
assertEquals(0.0, eval("0.0 || 0.0", bindings).value());
assertEquals(1L, eval("null || 1", bindings).value());
assertEquals(1L, eval("1 || null", bindings).value());
assertEquals(0L, eval("null || 0", bindings).value());
Assert.assertNull(eval("0 || null", bindings).value());
Assert.assertNull(eval("null || null", bindings).value());
Assert.assertNull(eval("null && 1", bindings).value());
Assert.assertNull(eval("1 && null", bindings).value());
Assert.assertNull(eval("null && 0", bindings).value());
assertEquals(0L, eval("0 && null", bindings).value());
assertNull(eval("null && null", bindings).value());
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
// in sql incompatible mode, null is false, so we return 0
assertEquals(NullHandling.defaultLongValue(), eval("null && 1", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("1 && null", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("null && null", bindings).valueOrDefault());
// if either side is false, output is false in both modes
assertEquals(0L, eval("null && 0", bindings).value());
assertEquals(0L, eval("0 && null", bindings).value());
}
@Test
@ -1143,86 +990,40 @@ public class EvalTest extends InitializedNullHandlingTest
bindingsMap.put("b2", false);
Expr.ObjectBinding bindings = InputBindings.forMap(bindingsMap);
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
assertEquals(1L, eval("s1 && s1", bindings).value());
assertEquals(0L, eval("s1 && s2", bindings).value());
assertEquals(0L, eval("s2 && s1", bindings).value());
assertEquals(0L, eval("s2 && s2", bindings).value());
assertEquals(1L, eval("s1 && s1", bindings).value());
assertEquals(0L, eval("s1 && s2", bindings).value());
assertEquals(0L, eval("s2 && s1", bindings).value());
assertEquals(0L, eval("s2 && s2", bindings).value());
assertEquals(1L, eval("s1 || s1", bindings).value());
assertEquals(1L, eval("s1 || s2", bindings).value());
assertEquals(1L, eval("s2 || s1", bindings).value());
assertEquals(0L, eval("s2 || s2", bindings).value());
assertEquals(1L, eval("s1 || s1", bindings).value());
assertEquals(1L, eval("s1 || s2", bindings).value());
assertEquals(1L, eval("s2 || s1", bindings).value());
assertEquals(0L, eval("s2 || s2", bindings).value());
assertEquals(1L, eval("l1 && l1", bindings).value());
assertEquals(0L, eval("l1 && l2", bindings).value());
assertEquals(0L, eval("l2 && l1", bindings).value());
assertEquals(0L, eval("l2 && l2", bindings).value());
assertEquals(1L, eval("l1 && l1", bindings).value());
assertEquals(0L, eval("l1 && l2", bindings).value());
assertEquals(0L, eval("l2 && l1", bindings).value());
assertEquals(0L, eval("l2 && l2", bindings).value());
assertEquals(1L, eval("b1 && b1", bindings).value());
assertEquals(0L, eval("b1 && b2", bindings).value());
assertEquals(0L, eval("b2 && b1", bindings).value());
assertEquals(0L, eval("b2 && b2", bindings).value());
assertEquals(1L, eval("b1 && b1", bindings).value());
assertEquals(0L, eval("b1 && b2", bindings).value());
assertEquals(0L, eval("b2 && b1", bindings).value());
assertEquals(0L, eval("b2 && b2", bindings).value());
assertEquals(1L, eval("d1 && d1", bindings).value());
assertEquals(0L, eval("d1 && d2", bindings).value());
assertEquals(0L, eval("d2 && d1", bindings).value());
assertEquals(0L, eval("d2 && d2", bindings).value());
assertEquals(1L, eval("d1 && d1", bindings).value());
assertEquals(0L, eval("d1 && d2", bindings).value());
assertEquals(0L, eval("d2 && d1", bindings).value());
assertEquals(0L, eval("d2 && d2", bindings).value());
assertEquals(1L, eval("b1", bindings).value());
assertEquals(1L, eval("if(b1,1,0)", bindings).value());
assertEquals(1L, eval("if(l1,1,0)", bindings).value());
assertEquals(1L, eval("if(d1,1,0)", bindings).value());
assertEquals(1L, eval("if(s1,1,0)", bindings).value());
assertEquals(0L, eval("if(b2,1,0)", bindings).value());
assertEquals(0L, eval("if(l2,1,0)", bindings).value());
assertEquals(0L, eval("if(d2,1,0)", bindings).value());
assertEquals(0L, eval("if(s2,1,0)", bindings).value());
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
try {
// turn on legacy insanity mode
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertEquals("true", eval("s1 && s1", bindings).value());
assertEquals("false", eval("s1 && s2", bindings).value());
assertEquals("false", eval("s2 && s1", bindings).value());
assertEquals("false", eval("s2 && s2", bindings).value());
assertEquals("true", eval("b1 && b1", bindings).value());
assertEquals("false", eval("b1 && b2", bindings).value());
assertEquals("false", eval("b2 && b1", bindings).value());
assertEquals("false", eval("b2 && b2", bindings).value());
assertEquals(100L, eval("l1 && l1", bindings).value());
assertEquals(0L, eval("l1 && l2", bindings).value());
assertEquals(0L, eval("l2 && l1", bindings).value());
assertEquals(0L, eval("l2 && l2", bindings).value());
assertEquals(1.1, eval("d1 && d1", bindings).value());
assertEquals(0.0, eval("d1 && d2", bindings).value());
assertEquals(0.0, eval("d2 && d1", bindings).value());
assertEquals(0.0, eval("d2 && d2", bindings).value());
assertEquals("true", eval("b1", bindings).value());
assertEquals(1L, eval("if(b1,1,0)", bindings).value());
assertEquals(1L, eval("if(l1,1,0)", bindings).value());
assertEquals(1L, eval("if(d1,1,0)", bindings).value());
assertEquals(1L, eval("if(s1,1,0)", bindings).value());
assertEquals(0L, eval("if(b2,1,0)", bindings).value());
assertEquals(0L, eval("if(l2,1,0)", bindings).value());
assertEquals(0L, eval("if(d2,1,0)", bindings).value());
assertEquals(0L, eval("if(s2,1,0)", bindings).value());
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
assertEquals(1L, eval("b1", bindings).value());
assertEquals(1L, eval("if(b1,1,0)", bindings).value());
assertEquals(1L, eval("if(l1,1,0)", bindings).value());
assertEquals(1L, eval("if(d1,1,0)", bindings).value());
assertEquals(1L, eval("if(s1,1,0)", bindings).value());
assertEquals(0L, eval("if(b2,1,0)", bindings).value());
assertEquals(0L, eval("if(l2,1,0)", bindings).value());
assertEquals(0L, eval("if(d2,1,0)", bindings).value());
assertEquals(0L, eval("if(s2,1,0)", bindings).value());
}
@Test
@ -1621,17 +1422,6 @@ public class EvalTest extends InitializedNullHandlingTest
assertBestEffortOf(true, ExpressionType.LONG, 1L);
assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L});
try {
// in non-strict boolean mode, they are strings
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertBestEffortOf(true, ExpressionType.STRING, "true");
assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"});
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
// doubles
assertBestEffortOf(1.0, ExpressionType.DOUBLE, 1.0);
assertBestEffortOf(1.0f, ExpressionType.DOUBLE, 1.0);

106
processing/src/test/java/org/apache/druid/math/expr/OutputTypeTest.java
View File

@ -70,29 +70,12 @@ public class OutputTypeTest extends InitializedNullHandlingTest
assertOutputType("-y", inspector, ExpressionType.LONG);
assertOutputType("-z", inspector, ExpressionType.DOUBLE);
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
assertOutputType("!'true'", inspector, ExpressionType.LONG);
assertOutputType("!1", inspector, ExpressionType.LONG);
assertOutputType("!x", inspector, ExpressionType.LONG);
assertOutputType("!y", inspector, ExpressionType.LONG);
assertOutputType("!1.1", inspector, ExpressionType.LONG);
assertOutputType("!z", inspector, ExpressionType.LONG);
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertOutputType("!1.1", inspector, ExpressionType.DOUBLE);
assertOutputType("!z", inspector, ExpressionType.DOUBLE);
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
assertOutputType("!'true'", inspector, ExpressionType.LONG);
assertOutputType("!1", inspector, ExpressionType.LONG);
assertOutputType("!x", inspector, ExpressionType.LONG);
assertOutputType("!y", inspector, ExpressionType.LONG);
assertOutputType("!1.1", inspector, ExpressionType.LONG);
assertOutputType("!z", inspector, ExpressionType.LONG);
}
@Test
@ -126,61 +109,32 @@ public class OutputTypeTest extends InitializedNullHandlingTest
assertOutputType("z^z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z%z_", inspector, ExpressionType.DOUBLE);
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
assertOutputType("y>y_", inspector, ExpressionType.LONG);
assertOutputType("y_<y", inspector, ExpressionType.LONG);
assertOutputType("y_<=y", inspector, ExpressionType.LONG);
assertOutputType("y_>=y", inspector, ExpressionType.LONG);
assertOutputType("y_==y", inspector, ExpressionType.LONG);
assertOutputType("y_!=y", inspector, ExpressionType.LONG);
assertOutputType("y_ && y", inspector, ExpressionType.LONG);
assertOutputType("y_ || y", inspector, ExpressionType.LONG);
assertOutputType("y>y_", inspector, ExpressionType.LONG);
assertOutputType("y_<y", inspector, ExpressionType.LONG);
assertOutputType("y_<=y", inspector, ExpressionType.LONG);
assertOutputType("y_>=y", inspector, ExpressionType.LONG);
assertOutputType("y_==y", inspector, ExpressionType.LONG);
assertOutputType("y_!=y", inspector, ExpressionType.LONG);
assertOutputType("y_ && y", inspector, ExpressionType.LONG);
assertOutputType("y_ || y", inspector, ExpressionType.LONG);
assertOutputType("z>y_", inspector, ExpressionType.LONG);
assertOutputType("z<y", inspector, ExpressionType.LONG);
assertOutputType("z<=y", inspector, ExpressionType.LONG);
assertOutputType("y>=z", inspector, ExpressionType.LONG);
assertOutputType("z==y", inspector, ExpressionType.LONG);
assertOutputType("z!=y", inspector, ExpressionType.LONG);
assertOutputType("z && y", inspector, ExpressionType.LONG);
assertOutputType("y || z", inspector, ExpressionType.LONG);
assertOutputType("z>y_", inspector, ExpressionType.LONG);
assertOutputType("z<y", inspector, ExpressionType.LONG);
assertOutputType("z<=y", inspector, ExpressionType.LONG);
assertOutputType("y>=z", inspector, ExpressionType.LONG);
assertOutputType("z==y", inspector, ExpressionType.LONG);
assertOutputType("z!=y", inspector, ExpressionType.LONG);
assertOutputType("z && y", inspector, ExpressionType.LONG);
assertOutputType("y || z", inspector, ExpressionType.LONG);
assertOutputType("z>z_", inspector, ExpressionType.LONG);
assertOutputType("z<z_", inspector, ExpressionType.LONG);
assertOutputType("z<=z_", inspector, ExpressionType.LONG);
assertOutputType("z_>=z", inspector, ExpressionType.LONG);
assertOutputType("z==z_", inspector, ExpressionType.LONG);
assertOutputType("z!=z_", inspector, ExpressionType.LONG);
assertOutputType("z && z_", inspector, ExpressionType.LONG);
assertOutputType("z_ || z", inspector, ExpressionType.LONG);
}
finally {
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertOutputType("z>y_", inspector, ExpressionType.DOUBLE);
assertOutputType("z<y", inspector, ExpressionType.DOUBLE);
assertOutputType("z<=y", inspector, ExpressionType.DOUBLE);
assertOutputType("y>=z", inspector, ExpressionType.DOUBLE);
assertOutputType("z==y", inspector, ExpressionType.DOUBLE);
assertOutputType("z!=y", inspector, ExpressionType.DOUBLE);
assertOutputType("z && y", inspector, ExpressionType.DOUBLE);
assertOutputType("y || z", inspector, ExpressionType.DOUBLE);
assertOutputType("z>z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z<z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z<=z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z_>=z", inspector, ExpressionType.DOUBLE);
assertOutputType("z==z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z!=z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z && z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z_ || z", inspector, ExpressionType.DOUBLE);
}
finally {
ExpressionProcessing.initializeForTests();
}
assertOutputType("z>z_", inspector, ExpressionType.LONG);
assertOutputType("z<z_", inspector, ExpressionType.LONG);
assertOutputType("z<=z_", inspector, ExpressionType.LONG);
assertOutputType("z_>=z", inspector, ExpressionType.LONG);
assertOutputType("z==z_", inspector, ExpressionType.LONG);
assertOutputType("z!=z_", inspector, ExpressionType.LONG);
assertOutputType("z && z_", inspector, ExpressionType.LONG);
assertOutputType("z_ || z", inspector, ExpressionType.LONG);
assertOutputType("1*(2 + 3.0)", inspector, ExpressionType.DOUBLE);
}

71
processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
View File

@ -30,7 +30,6 @@ import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.Druids;
import org.apache.druid.query.NestedDataTestUtils;
import org.apache.druid.query.Query;
@ -668,76 +667,6 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
}
@Test
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesNonStrictBooleans() throws Exception
{
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
.dataSource("test_datasource")
.intervals(
new MultipleIntervalSegmentSpec(
Collections.singletonList(Intervals.ETERNITY)
)
)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.limit(100)
.context(ImmutableMap.of());
Query<ScanResultValue> scanQuery = builder.build();
final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")};
List<Segment> realtimeSegs = ImmutableList.of(
NestedDataTestUtils.createIncrementalIndex(
tempFolder,
NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
TestIndex.DEFAULT_JSON_INPUT_FORMAT,
NestedDataTestUtils.TIMESTAMP_SPEC,
NestedDataTestUtils.AUTO_DISCOVERY,
TransformSpec.NONE,
aggs,
Granularities.NONE,
true
)
);
List<Segment> segs = NestedDataTestUtils.createSegments(
tempFolder,
closer,
NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
TestIndex.DEFAULT_JSON_INPUT_FORMAT,
NestedDataTestUtils.TIMESTAMP_SPEC,
NestedDataTestUtils.AUTO_DISCOVERY,
TransformSpec.NONE,
aggs,
Granularities.NONE,
true,
IndexSpec.DEFAULT
);
final Sequence<ScanResultValue> seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery);
final Sequence<ScanResultValue> seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery);
List<ScanResultValue> resultsRealtime = seq.toList();
List<ScanResultValue> resultsSegments = seq2.toList();
logResults(resultsSegments);
logResults(resultsRealtime);
Assert.assertEquals(1, resultsRealtime.size());
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
Assert.assertEquals(
"["
+ "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
resultsSegments.get(0).getEvents().toString()
);
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
}
finally {
ExpressionProcessing.initializeForTests();
}
}
@Test
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception
{

143
processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java
View File

@ -1,143 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.filter;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.filter.NotDimFilter;
import org.apache.druid.segment.CursorFactory;
import org.apache.druid.segment.IndexBuilder;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.Closeable;
@RunWith(Parameterized.class)
public class ExpressionFilterNonStrictBooleansTest extends ExpressionFilterTest
{
public ExpressionFilterNonStrictBooleansTest(
String testName,
IndexBuilder indexBuilder,
Function<IndexBuilder, Pair<CursorFactory, Closeable>> finisher,
boolean cnf,
boolean optimize
)
{
super(testName, indexBuilder, finisher, cnf, optimize);
}
@Before
@Override
public void setup()
{
ExpressionProcessing.initializeForStrictBooleansTests(false);
}
@Override
@Test
public void testComplement()
{
if (NullHandling.sqlCompatible()) {
assertFilterMatches(edf("dim5 == 'a'"), ImmutableList.of("0"));
// non-strict mode is wild
assertFilterMatches(
NotDimFilter.of(edf("dim5 == 'a'")),
ImmutableList.of("1", "2", "3", "4", "5", "6", "7", "8", "9")
);
assertFilterMatches(
edf("dim5 == ''"), ImmutableList.of("4")
);
// non-strict mode!
assertFilterMatches(
NotDimFilter.of(edf("dim5 == ''")), ImmutableList.of("0", "1", "2", "3", "5", "6", "7", "8", "9")
);
} else {
assertFilterMatches(edf("dim5 == 'a'"), ImmutableList.of("0"));
assertFilterMatches(
NotDimFilter.of(edf("dim5 == 'a'")),
ImmutableList.of("1", "2", "3", "4", "5", "6", "7", "8", "9")
);
}
}
@Override
@Test
public void testMissingColumn()
{
if (NullHandling.replaceWithDefault()) {
assertFilterMatches(
edf("missing == ''"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
assertFilterMatches(
edf("missing == otherMissing"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
} else {
// AS per SQL standard null == null returns false.
assertFilterMatches(edf("missing == null"), ImmutableList.of());
// in non-strict mode, madness happens
assertFilterMatches(
NotDimFilter.of(edf("missing == null")),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
// also this madness doesn't do madness
assertFilterMatches(
edf("missing == otherMissing"),
ImmutableList.of()
);
assertFilterMatches(
NotDimFilter.of(edf("missing == otherMissing")),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
}
assertFilterMatches(edf("missing == '1'"), ImmutableList.of());
assertFilterMatches(edf("missing == 2"), ImmutableList.of());
if (NullHandling.replaceWithDefault()) {
// missing equivaluent to 0
assertFilterMatches(
edf("missing < '2'"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
assertFilterMatches(
edf("missing < 2"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
assertFilterMatches(
edf("missing < 2.0"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
} else {
// missing equivalent to null
assertFilterMatches(edf("missing < '2'"), ImmutableList.of());
assertFilterMatches(edf("missing < 2"), ImmutableList.of());
assertFilterMatches(edf("missing < 2.0"), ImmutableList.of());
}
assertFilterMatches(edf("missing > '2'"), ImmutableList.of());
assertFilterMatches(edf("missing > 2"), ImmutableList.of());
assertFilterMatches(edf("missing > 2.0"), ImmutableList.of());
assertFilterMatchesSkipVectorize(edf("like(missing, '1%')"), ImmutableList.of());
}
}

17
processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
View File

@ -36,7 +36,6 @@ import org.apache.druid.data.input.impl.TimeAndDimsParseSpec;
import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.filter.ExpressionDimFilter;
import org.apache.druid.query.filter.Filter;
@ -46,10 +45,8 @@ import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@ -126,22 +123,10 @@ public class ExpressionFilterTest extends BaseFilterTest
);
}
@Before
public void setup()
{
ExpressionProcessing.initializeForStrictBooleansTests(true);
}
@After
public void teardown()
{
ExpressionProcessing.initializeForTests();
}
@AfterClass
public static void tearDown() throws Exception
{
BaseFilterTest.tearDown(ColumnComparisonFilterTest.class.getName());
BaseFilterTest.tearDown(ExpressionFilterTest.class.getName());
}
@Test

78
processing/src/test/java/org/apache/druid/segment/transform/TransformSpecTest.java
View File

@ -30,7 +30,6 @@ import org.apache.druid.data.input.impl.MapInputRowParser;
import org.apache.druid.data.input.impl.TimeAndDimsParseSpec;
import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.filter.AndDimFilter;
import org.apache.druid.query.filter.SelectorDimFilter;
@ -208,68 +207,31 @@ public class TransformSpecTest extends InitializedNullHandlingTest
@Test
public void testBoolTransforms()
{
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
final TransformSpec transformSpec = new TransformSpec(
null,
ImmutableList.of(
new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE),
new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE)
)
);
final TransformSpec transformSpec = new TransformSpec(
null,
ImmutableList.of(
new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE),
new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE)
)
);
Assert.assertEquals(
ImmutableSet.of("bool"),
transformSpec.getRequiredColumns()
);
Assert.assertEquals(
ImmutableSet.of("bool"),
transformSpec.getRequiredColumns()
);
final InputRowParser<Map<String, Object>> parser = transformSpec.decorate(PARSER);
final InputRow row = parser.parseBatch(ROW1).get(0);
final InputRowParser<Map<String, Object>> parser = transformSpec.decorate(PARSER);
final InputRow row = parser.parseBatch(ROW1).get(0);
Assert.assertNotNull(row);
Assert.assertEquals(1L, row.getRaw("truthy1"));
Assert.assertEquals(1L, row.getRaw("truthy2"));
Assert.assertNotNull(row);
Assert.assertEquals(1L, row.getRaw("truthy1"));
Assert.assertEquals(1L, row.getRaw("truthy2"));
final InputRow row2 = parser.parseBatch(ROW2).get(0);
final InputRow row2 = parser.parseBatch(ROW2).get(0);
Assert.assertNotNull(row2);
Assert.assertEquals(0L, row2.getRaw("truthy1"));
Assert.assertEquals(0L, row2.getRaw("truthy2"));
}
finally {
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
final TransformSpec transformSpec = new TransformSpec(
null,
ImmutableList.of(
new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE),
new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE)
)
);
Assert.assertEquals(
ImmutableSet.of("bool"),
transformSpec.getRequiredColumns()
);
final InputRowParser<Map<String, Object>> parser = transformSpec.decorate(PARSER);
final InputRow row = parser.parseBatch(ROW1).get(0);
Assert.assertNotNull(row);
Assert.assertEquals("true", row.getRaw("truthy1"));
Assert.assertEquals(1L, row.getRaw("truthy2"));
final InputRow row2 = parser.parseBatch(ROW2).get(0);
Assert.assertNotNull(row2);
Assert.assertEquals("false", row2.getRaw("truthy1"));
Assert.assertEquals(0L, row2.getRaw("truthy2"));
}
finally {
ExpressionProcessing.initializeForTests();
}
Assert.assertNotNull(row2);
Assert.assertEquals(0L, row2.getRaw("truthy1"));
Assert.assertEquals(0L, row2.getRaw("truthy2"));
}
@Test

3
sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRule.java
View File

@ -27,7 +27,6 @@ import org.apache.calcite.rel.logical.LogicalValues;
import org.apache.calcite.rex.RexLiteral;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.error.InvalidSqlInput;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.InlineDataSource;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.planner.Calcites;
@ -122,7 +121,7 @@ public class DruidLogicalValuesRule extends RelOptRule
}
return ((Number) RexLiteral.value(literal)).longValue();
case BOOLEAN:
if (ExpressionProcessing.useStrictBooleans() && NullHandling.sqlCompatible() && literal.isNull()) {
if (NullHandling.sqlCompatible() && literal.isNull()) {
return null;
}
return literal.isAlwaysTrue() ? 1L : 0L;

3
sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java
View File

@ -31,7 +31,6 @@ import org.apache.calcite.util.TimestampString;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.error.DruidExceptionMatcher;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.sql.calcite.planner.DruidTypeSystem;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.testing.InitializedNullHandlingTest;
@ -144,7 +143,7 @@ public class DruidLogicalValuesRuleTest
{
RexLiteral literal = REX_BUILDER.makeLiteral(null, REX_BUILDER.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN));
if (NullHandling.sqlCompatible() && ExpressionProcessing.useStrictBooleans()) {
if (NullHandling.sqlCompatible()) {
final Object fromLiteral = DruidLogicalValuesRule.getValueFromLiteral(literal, DEFAULT_CONTEXT);
Assert.assertNull(fromLiteral);
} else {