Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove druid.expressions.useStrictBooleans in favor of always being true (#17568)

This commit is contained in:
Clint Wylie 2024-12-17 18:49:16 -08:00 committed by GitHub
parent 8b81c91979
commit a44ab109d5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 206 additions and 1364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java
View File

@ -31,7 +31,6 @@ import org.apache.druid.data.input.impl.LongDimensionSchema;
import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.StringDimensionSchema;
import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.segment.AutoTypeColumnSchema; import org.apache.druid.segment.AutoTypeColumnSchema;
import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
@ -56,71 +55,6 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest
); );
private InputSourceSampler inputSourceSampler = new InputSourceSampler(OBJECT_MAPPER); private InputSourceSampler inputSourceSampler = new InputSourceSampler(OBJECT_MAPPER);
@Test
public void testDiscoveredTypesNonStrictBooleans()
{
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
final SamplerResponse response = inputSourceSampler.sample(
inputSource,
new JsonInputFormat(null, null, null, null, null),
DataSchema.builder()
.withDataSource("test")
.withTimestamp(new TimestampSpec("t", null, null))
.withDimensions(DimensionsSpec.builder().useSchemaDiscovery(true).build())
.build(),
null
);
Assert.assertEquals(6, response.getNumRowsRead());
Assert.assertEquals(5, response.getNumRowsIndexed());
Assert.assertEquals(6, response.getData().size());
Assert.assertEquals(
ImmutableList.of(
new StringDimensionSchema("string"),
new LongDimensionSchema("long"),
new DoubleDimensionSchema("double"),
new StringDimensionSchema("bool"),
new StringDimensionSchema("variant"),
new AutoTypeColumnSchema("array", null),
new AutoTypeColumnSchema("nested", null)
),
response.getLogicalDimensions()
);
Assert.assertEquals(
ImmutableList.of(
new AutoTypeColumnSchema("string", null),
new AutoTypeColumnSchema("long", null),
new AutoTypeColumnSchema("double", null),
new AutoTypeColumnSchema("bool", null),
new AutoTypeColumnSchema("variant", null),
new AutoTypeColumnSchema("array", null),
new AutoTypeColumnSchema("nested", null)
),
response.getPhysicalDimensions()
);
Assert.assertEquals(
RowSignature.builder()
.addTimeColumn()
.add("string", ColumnType.STRING)
.add("long", ColumnType.LONG)
.add("double", ColumnType.DOUBLE)
.add("bool", ColumnType.STRING)
.add("variant", ColumnType.STRING)
.add("array", ColumnType.LONG_ARRAY)
.add("nested", ColumnType.NESTED_DATA)
.build(),
response.getLogicalSegmentSchema()
);
}
finally {
ExpressionProcessing.initializeForTests();
}
}
@Test @Test
public void testDiscoveredTypesStrictBooleans() public void testDiscoveredTypesStrictBooleans()
{ {

4
processing/src/main/java/org/apache/druid/common/config/NullHandling.java
View File

@ -22,7 +22,6 @@ package org.apache.druid.common.config;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.inject.Inject; import com.google.inject.Inject;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.filter.DimFilter;
import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.filter.ValueMatcher;
@ -130,8 +129,7 @@ public class NullHandling
public static boolean useThreeValueLogic() public static boolean useThreeValueLogic()
{ {
return sqlCompatible() && return sqlCompatible() &&
INSTANCE.isUseThreeValueLogicForNativeFilters() && INSTANCE.isUseThreeValueLogicForNativeFilters();
ExpressionProcessing.useStrictBooleans();
} }
@Nullable @Nullable

10
processing/src/main/java/org/apache/druid/math/expr/BinaryEvalOpExprBase.java
View File

@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableSet;
import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.column.Types;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.util.Objects; import java.util.Objects;
@ -206,9 +205,6 @@ abstract class BinaryBooleanOpExprBase extends BinaryOpExprBase
result = evalDouble(leftVal.asDouble(), rightVal.asDouble()); result = evalDouble(leftVal.asDouble(), rightVal.asDouble());
break; break;
} }
if (!ExpressionProcessing.useStrictBooleans() && !type.is(ExprType.STRING) && !type.isArray()) {
return ExprEval.ofBoolean(result, type);
}
return ExprEval.ofLongBoolean(result); return ExprEval.ofLongBoolean(result);
} }
@ -224,11 +220,7 @@ abstract class BinaryBooleanOpExprBase extends BinaryOpExprBase
@Override @Override
public ExpressionType getOutputType(InputBindingInspector inspector) public ExpressionType getOutputType(InputBindingInspector inspector)
{ {
ExpressionType implicitCast = super.getOutputType(inspector); return ExpressionType.LONG;
if (ExpressionProcessing.useStrictBooleans() || Types.isNullOr(implicitCast, ExprType.STRING)) {
return ExpressionType.LONG;
}
return implicitCast;
} }
@Override @Override

20
processing/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java
View File

@ -340,9 +340,6 @@ class BinAndExpr extends BinaryOpExprBase
public ExprEval eval(ObjectBinding bindings) public ExprEval eval(ObjectBinding bindings)
{ {
ExprEval leftVal = left.eval(bindings); ExprEval leftVal = left.eval(bindings);
if (!ExpressionProcessing.useStrictBooleans()) {
return leftVal.asBoolean() ? right.eval(bindings) : leftVal;
}
// if left is false, always false // if left is false, always false
if (leftVal.value() != null && !leftVal.asBoolean()) { if (leftVal.value() != null && !leftVal.asBoolean()) {
@ -376,9 +373,7 @@ class BinAndExpr extends BinaryOpExprBase
@Override @Override
public boolean canVectorize(InputBindingInspector inspector) public boolean canVectorize(InputBindingInspector inspector)
{ {
return ExpressionProcessing.useStrictBooleans() && return inspector.areSameTypes(left, right) && inspector.canVectorize(left, right);
inspector.areSameTypes(left, right) &&
inspector.canVectorize(left, right);
} }
@Override @Override
@ -391,9 +386,6 @@ class BinAndExpr extends BinaryOpExprBase
@Override @Override
public ExpressionType getOutputType(InputBindingInspector inspector) public ExpressionType getOutputType(InputBindingInspector inspector)
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
return super.getOutputType(inspector);
}
return ExpressionType.LONG; return ExpressionType.LONG;
} }
} }
@ -415,9 +407,6 @@ class BinOrExpr extends BinaryOpExprBase
public ExprEval eval(ObjectBinding bindings) public ExprEval eval(ObjectBinding bindings)
{ {
ExprEval leftVal = left.eval(bindings); ExprEval leftVal = left.eval(bindings);
if (!ExpressionProcessing.useStrictBooleans()) {
return leftVal.asBoolean() ? leftVal : right.eval(bindings);
}
// if left is true, always true // if left is true, always true
if (leftVal.value() != null && leftVal.asBoolean()) { if (leftVal.value() != null && leftVal.asBoolean()) {
@ -454,9 +443,7 @@ class BinOrExpr extends BinaryOpExprBase
public boolean canVectorize(InputBindingInspector inspector) public boolean canVectorize(InputBindingInspector inspector)
{ {
return ExpressionProcessing.useStrictBooleans() && return inspector.areSameTypes(left, right) && inspector.canVectorize(left, right);
inspector.areSameTypes(left, right) &&
inspector.canVectorize(left, right);
} }
@Override @Override
@ -469,9 +456,6 @@ class BinOrExpr extends BinaryOpExprBase
@Override @Override
public ExpressionType getOutputType(InputBindingInspector inspector) public ExpressionType getOutputType(InputBindingInspector inspector)
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
return super.getOutputType(inspector);
}
return ExpressionType.LONG; return ExpressionType.LONG;
} }
} }

33
processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
View File

@ -242,11 +242,7 @@ public abstract class ExprEval<T>
if (Number.class.isAssignableFrom(next) || next == String.class || next == Boolean.class) { if (Number.class.isAssignableFrom(next) || next == String.class || next == Boolean.class) {
// coerce booleans // coerce booleans
if (next == Boolean.class) { if (next == Boolean.class) {
if (ExpressionProcessing.useStrictBooleans()) { next = Long.class;
next = Long.class;
} else {
next = String.class;
}
} }
if (existing == null) { if (existing == null) {
return next; return next;
@ -350,28 +346,6 @@ public abstract class ExprEval<T>
return new ArrayExprEval(outputType, value); return new ArrayExprEval(outputType, value);
} }
/**
* Convert a boolean back into native expression type
*
* Do not use this method unless {@link ExpressionProcessing#useStrictBooleans()} is set to false.
* {@link ExpressionType#LONG} is the Druid boolean unless this mode is enabled, so use {@link #ofLongBoolean}
* instead.
*/
@Deprecated
public static ExprEval ofBoolean(boolean value, ExpressionType type)
{
switch (type.getType()) {
case DOUBLE:
return of(Evals.asDouble(value));
case LONG:
return ofLongBoolean(value);
case STRING:
return of(String.valueOf(value));
default:
throw new Types.InvalidCastBooleanException(type);
}
}
/** /**
* Convert a boolean into a long expression type * Convert a boolean into a long expression type
*/ */
@ -421,10 +395,7 @@ public abstract class ExprEval<T>
return new LongExprEval((Number) val); return new LongExprEval((Number) val);
} }
if (val instanceof Boolean) { if (val instanceof Boolean) {
if (ExpressionProcessing.useStrictBooleans()) { return ofLongBoolean((Boolean) val);
return ofLongBoolean((Boolean) val);
}
return new StringExprEval(String.valueOf(val));
} }
if (val instanceof Long[]) { if (val instanceof Long[]) {
final Long[] inputArray = (Long[]) val; final Long[] inputArray = (Long[]) val;

15
processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java
View File

@ -48,12 +48,6 @@ public class ExpressionProcessing
INSTANCE = new ExpressionProcessingConfig(null, null, null, null); INSTANCE = new ExpressionProcessingConfig(null, null, null, null);
} }
@VisibleForTesting
public static void initializeForStrictBooleansTests(boolean useStrict)
{
INSTANCE = new ExpressionProcessingConfig(useStrict, null, null, null);
}
@VisibleForTesting @VisibleForTesting
public static void initializeForHomogenizeNullMultiValueStrings() public static void initializeForHomogenizeNullMultiValueStrings()
{ {
@ -66,15 +60,6 @@ public class ExpressionProcessing
INSTANCE = new ExpressionProcessingConfig(null, null, null, true); INSTANCE = new ExpressionProcessingConfig(null, null, null, true);
} }
/**
* All boolean expressions are {@link ExpressionType#LONG}
*/
public static boolean useStrictBooleans()
{
checkInitialized();
return INSTANCE.isUseStrictBooleans();
}
/** /**
* All {@link ExprType#ARRAY} values will be converted to {@link ExpressionType#STRING} by their column selectors * All {@link ExprType#ARRAY} values will be converted to {@link ExpressionType#STRING} by their column selectors
* (not within expression processing) to be treated as multi-value strings instead of native arrays. * (not within expression processing) to be treated as multi-value strings instead of native arrays.

17
processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
View File

@ -30,6 +30,7 @@ public class ExpressionProcessingConfig
{ {
private static final Logger LOG = new Logger(ExpressionProcessingConfig.class); private static final Logger LOG = new Logger(ExpressionProcessingConfig.class);
@Deprecated
public static final String NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING = "druid.expressions.useStrictBooleans"; public static final String NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING = "druid.expressions.useStrictBooleans";
// Coerce arrays to multi value strings // Coerce arrays to multi value strings
public static final String PROCESS_ARRAYS_AS_MULTIVALUE_STRINGS_CONFIG_STRING = public static final String PROCESS_ARRAYS_AS_MULTIVALUE_STRINGS_CONFIG_STRING =
@ -39,9 +40,6 @@ public class ExpressionProcessingConfig
"druid.expressions.homogenizeNullMultiValueStringArrays"; "druid.expressions.homogenizeNullMultiValueStringArrays";
public static final String ALLOW_VECTORIZE_FALLBACK = "druid.expressions.allowVectorizeFallback"; public static final String ALLOW_VECTORIZE_FALLBACK = "druid.expressions.allowVectorizeFallback";
@JsonProperty("useStrictBooleans")
private final boolean useStrictBooleans;
@JsonProperty("processArraysAsMultiValueStrings") @JsonProperty("processArraysAsMultiValueStrings")
private final boolean processArraysAsMultiValueStrings; private final boolean processArraysAsMultiValueStrings;
@ -51,9 +49,13 @@ public class ExpressionProcessingConfig
@JsonProperty("allowVectorizeFallback") @JsonProperty("allowVectorizeFallback")
private final boolean allowVectorizeFallback; private final boolean allowVectorizeFallback;
@Deprecated
@JsonProperty("useStrictBooleans")
private final boolean useStrictBooleans;
@JsonCreator @JsonCreator
public ExpressionProcessingConfig( public ExpressionProcessingConfig(
@JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans, @Deprecated @JsonProperty("useStrictBooleans") @Nullable Boolean useStrictBooleans,
@JsonProperty("processArraysAsMultiValueStrings") @Nullable Boolean processArraysAsMultiValueStrings, @JsonProperty("processArraysAsMultiValueStrings") @Nullable Boolean processArraysAsMultiValueStrings,
@JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays, @JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays,
@JsonProperty("allowVectorizeFallback") @Nullable Boolean allowVectorizeFallback @JsonProperty("allowVectorizeFallback") @Nullable Boolean allowVectorizeFallback
@ -83,17 +85,12 @@ public class ExpressionProcessingConfig
final String docsBaseFormat = "https://druid.apache.org/docs/%s/querying/sql-data-types#%s"; final String docsBaseFormat = "https://druid.apache.org/docs/%s/querying/sql-data-types#%s";
if (!this.useStrictBooleans) { if (!this.useStrictBooleans) {
LOG.warn( LOG.warn(
"druid.expressions.useStrictBooleans set to 'false', we recommend using 'true' if using SQL to query Druid for the most SQL compliant behavior, see %s for details", "druid.expressions.useStrictBooleans set to 'false', but has been removed from Druid and is always 'true' now for the most SQL compliant behavior, see %s for details",
StringUtils.format(docsBaseFormat, version, "boolean-logic") StringUtils.format(docsBaseFormat, version, "boolean-logic")
); );
} }
} }
public boolean isUseStrictBooleans()
{
return useStrictBooleans;
}
public boolean processArraysAsMultiValueStrings() public boolean processArraysAsMultiValueStrings()
{ {
return processArraysAsMultiValueStrings; return processArraysAsMultiValueStrings;

13
processing/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java
View File

@ -26,7 +26,6 @@ import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.math.expr.vector.ExprVectorProcessor;
import org.apache.druid.math.expr.vector.VectorMathProcessors; import org.apache.druid.math.expr.vector.VectorMathProcessors;
import org.apache.druid.math.expr.vector.VectorProcessors; import org.apache.druid.math.expr.vector.VectorProcessors;
import org.apache.druid.segment.column.Types;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.math.BigInteger; import java.math.BigInteger;
@ -181,11 +180,6 @@ class UnaryNotExpr extends UnaryExpr
if (NullHandling.sqlCompatible() && (ret.value() == null)) { if (NullHandling.sqlCompatible() && (ret.value() == null)) {
return ExprEval.of(null); return ExprEval.of(null);
} }
if (!ExpressionProcessing.useStrictBooleans()) {
// conforming to other boolean-returning binary operators
ExpressionType retType = ret.type().is(ExprType.DOUBLE) ? ExpressionType.DOUBLE : ExpressionType.LONG;
return ExprEval.ofBoolean(!ret.asBoolean(), retType);
}
return ExprEval.ofLongBoolean(!ret.asBoolean()); return ExprEval.ofLongBoolean(!ret.asBoolean());
} }
@ -193,13 +187,6 @@ class UnaryNotExpr extends UnaryExpr
@Override @Override
public ExpressionType getOutputType(InputBindingInspector inspector) public ExpressionType getOutputType(InputBindingInspector inspector)
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
ExpressionType implicitCast = super.getOutputType(inspector);
if (Types.is(implicitCast, ExprType.STRING)) {
return ExpressionType.LONG;
}
return implicitCast;
}
return ExpressionType.LONG; return ExpressionType.LONG;
} }

467
processing/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java
View File

@ -23,7 +23,6 @@ import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.ExprType;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.segment.column.Types; import org.apache.druid.segment.column.Types;
@ -33,50 +32,6 @@ import java.util.function.Supplier;
public class VectorComparisonProcessors public class VectorComparisonProcessors
{ {
@Deprecated
public static <T> ExprVectorProcessor<T> makeComparisonProcessor(
Expr.VectorInputBindingInspector inspector,
Expr left,
Expr right,
Supplier<LongOutObjectsInFunctionVectorProcessor> longOutStringsInFunctionVectorProcessor,
Supplier<LongOutLongsInFunctionVectorValueProcessor> longOutLongsInProcessor,
Supplier<DoubleOutLongDoubleInFunctionVectorValueProcessor> doubleOutLongDoubleInProcessor,
Supplier<DoubleOutDoubleLongInFunctionVectorValueProcessor> doubleOutDoubleLongInProcessor,
Supplier<DoubleOutDoublesInFunctionVectorValueProcessor> doubleOutDoublesInProcessor
)
{
assert !ExpressionProcessing.useStrictBooleans();
final ExpressionType leftType = left.getOutputType(inspector);
final ExpressionType rightType = right.getOutputType(inspector);
ExprVectorProcessor<?> processor = null;
if (Types.is(leftType, ExprType.STRING)) {
if (Types.isNullOr(rightType, ExprType.STRING)) {
processor = longOutStringsInFunctionVectorProcessor.get();
} else {
processor = doubleOutDoublesInProcessor.get();
}
} else if (leftType == null) {
if (Types.isNullOr(rightType, ExprType.STRING)) {
processor = longOutStringsInFunctionVectorProcessor.get();
}
} else if (leftType.is(ExprType.DOUBLE) || Types.is(rightType, ExprType.DOUBLE)) {
processor = doubleOutDoublesInProcessor.get();
}
if (processor != null) {
return (ExprVectorProcessor<T>) processor;
}
// fall through to normal math processor logic
return VectorMathProcessors.makeMathProcessor(
inspector,
left,
right,
longOutLongsInProcessor,
doubleOutLongDoubleInProcessor,
doubleOutDoubleLongInProcessor,
doubleOutDoublesInProcessor
);
}
public static <T> ExprVectorProcessor<T> makeBooleanProcessor( public static <T> ExprVectorProcessor<T> makeBooleanProcessor(
Expr.VectorInputBindingInspector inspector, Expr.VectorInputBindingInspector inspector,
Expr left, Expr left,
@ -131,75 +86,6 @@ public class VectorComparisonProcessors
Expr right Expr right
) )
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(Objects.equals(leftVal, rightVal));
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left == right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(left == right);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(left == right);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(left == right);
}
}
);
}
return makeBooleanProcessor( return makeBooleanProcessor(
inspector, inspector,
left, left,
@ -275,75 +161,6 @@ public class VectorComparisonProcessors
Expr right Expr right
) )
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(!Objects.equals(leftVal, rightVal));
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left != right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(left != right);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(left != right);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(left != right);
}
}
);
}
return makeBooleanProcessor( return makeBooleanProcessor(
inspector, inspector,
left, left,
@ -419,77 +236,6 @@ public class VectorComparisonProcessors
Expr right Expr right
) )
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) >= 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left >= right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(Double.compare(left, right) >= 0);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(Double.compare(left, right) >= 0);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(Double.compare(left, right) >= 0);
}
}
);
}
return makeBooleanProcessor( return makeBooleanProcessor(
inspector, inspector,
left, left,
@ -567,77 +313,6 @@ public class VectorComparisonProcessors
Expr right Expr right
) )
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) > 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left > right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(Double.compare(left, right) > 0);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(Double.compare(left, right) > 0);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(Double.compare(left, right) > 0);
}
}
);
}
return makeBooleanProcessor( return makeBooleanProcessor(
inspector, inspector,
left, left,
@ -715,77 +390,6 @@ public class VectorComparisonProcessors
Expr right Expr right
) )
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) <= 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left <= right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(Double.compare(left, right) <= 0);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(Double.compare(left, right) <= 0);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(Double.compare(left, right) <= 0);
}
}
);
}
return makeBooleanProcessor( return makeBooleanProcessor(
inspector, inspector,
left, left,
@ -863,77 +467,6 @@ public class VectorComparisonProcessors
Expr right Expr right
) )
{ {
if (!ExpressionProcessing.useStrictBooleans()) {
return makeComparisonProcessor(
inspector,
left,
right,
() -> new LongOutObjectsInFunctionVectorProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize(),
ExpressionType.STRING
)
{
@Nullable
@Override
Long processValue(@Nullable Object leftVal, @Nullable Object rightVal)
{
return Evals.asLong(
Comparators.<String>naturalNullsFirst().compare((String) leftVal, (String) rightVal) < 0
);
}
},
() -> new LongOutLongsInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public long apply(long left, long right)
{
return Evals.asLong(left < right);
}
},
() -> new DoubleOutLongDoubleInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(long left, double right)
{
return Evals.asDouble(Double.compare(left, right) < 0);
}
},
() -> new DoubleOutDoubleLongInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, long right)
{
return Evals.asDouble(Double.compare(left, right) < 0);
}
},
() -> new DoubleOutDoublesInFunctionVectorValueProcessor(
left.asVectorProcessor(inspector),
right.asVectorProcessor(inspector),
inspector.getMaxVectorSize()
)
{
@Override
public double apply(double left, double right)
{
return Evals.asDouble(Double.compare(left, right) < 0);
}
}
);
}
return makeBooleanProcessor( return makeBooleanProcessor(
inspector, inspector,
left, left,

26
processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java
View File

@ -25,7 +25,6 @@ import org.apache.druid.error.DruidException;
import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.ExprType;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.Exprs; import org.apache.druid.math.expr.Exprs;
import org.apache.druid.segment.column.Types; import org.apache.druid.segment.column.Types;
@ -658,25 +657,14 @@ public class VectorProcessors
} }
}; };
} else if (Types.is(inputType, ExprType.DOUBLE)) { } else if (Types.is(inputType, ExprType.DOUBLE)) {
if (!ExpressionProcessing.useStrictBooleans()) { processor = new LongOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize)
processor = new DoubleOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize) {
@Override
public long apply(double input)
{ {
@Override return Evals.asLong(!Evals.asBoolean(input));
public double apply(double input) }
{ };
return Evals.asDouble(!Evals.asBoolean(input));
}
};
} else {
processor = new LongOutDoubleInFunctionVectorValueProcessor(expr.asVectorProcessor(inspector), maxVectorSize)
{
@Override
public long apply(double input)
{
return Evals.asLong(!Evals.asBoolean(input));
}
};
}
} }
if (processor == null) { if (processor == null) {
throw Exprs.cannotVectorize(); throw Exprs.cannotVectorize();

10
processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java
View File

@ -21,7 +21,6 @@ package org.apache.druid.segment.filter;
import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.BitmapResultFactory; import org.apache.druid.query.BitmapResultFactory;
import org.apache.druid.query.filter.ColumnIndexSelector; import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.Filter; import org.apache.druid.query.filter.Filter;
@ -45,11 +44,10 @@ import java.util.Set;
/** /**
* Nice filter you have there... NOT! * Nice filter you have there... NOT!
* *
* If {@link ExpressionProcessing#useStrictBooleans()} and {@link NullHandling#sqlCompatible()} are both true, this * If {@link NullHandling#sqlCompatible()} is true, this filter inverts the {@code includeUnknown} flag to properly
* filter inverts the {@code includeUnknown} flag to properly map Druids native two-valued logic (true, false) to SQL * map Druids native two-valued logic (true, false) to SQL three-valued logic (true, false, unknown). At the top level,
* three-valued logic (true, false, unknown). At the top level, this flag is always passed in as 'false', and is only * this flag is always passed in as 'false', and is only flipped by this filter. Other logical filters
* flipped by this filter. Other logical filters ({@link AndFilter} and {@link OrFilter}) propagate the value of * ({@link AndFilter} and {@link OrFilter}) propagate the value of {@code includeUnknown} to their children.
* {@code includeUnknown} to their children.
* *
* For example, if the base filter is equality, by default value matchers and indexes only return true for the rows * For example, if the base filter is equality, by default value matchers and indexes only return true for the rows
* that are equal to the value. When wrapped in a not filter, the not filter indicates that the equality matchers and * that are equal to the value. When wrapped in a not filter, the not filter indicates that the equality matchers and

468
processing/src/test/java/org/apache/druid/math/expr/EvalTest.java
View File

@ -41,7 +41,6 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
/** /**
*/ */
@ -84,93 +83,50 @@ public class EvalTest extends InitializedNullHandlingTest
assertEquals(2.0, evalDouble("\"x\"", bindings), 0.0001); assertEquals(2.0, evalDouble("\"x\"", bindings), 0.0001);
assertEquals(304.0, evalDouble("300 + \"x\" * 2", bindings), 0.0001); assertEquals(304.0, evalDouble("300 + \"x\" * 2", bindings), 0.0001);
try { Assert.assertEquals(0L, evalLong("1.0 && 0.0", bindings));
ExpressionProcessing.initializeForStrictBooleansTests(false); Assert.assertEquals(1L, evalLong("1.0 && 2.0", bindings));
Assert.assertFalse(evalDouble("1.0 && 0.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("1.0 && 2.0", bindings) > 0.0);
Assert.assertTrue(evalDouble("1.0 || 0.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("1.0 || 0.0", bindings));
Assert.assertFalse(evalDouble("0.0 || 0.0", bindings) > 0.0); Assert.assertEquals(0L, evalLong("0.0 || 0.0", bindings));
Assert.assertTrue(evalDouble("2.0 > 1.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("2.0 > 1.0", bindings));
Assert.assertTrue(evalDouble("2.0 >= 2.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("2.0 >= 2.0", bindings));
Assert.assertTrue(evalDouble("1.0 < 2.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("1.0 < 2.0", bindings));
Assert.assertTrue(evalDouble("2.0 <= 2.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("2.0 <= 2.0", bindings));
Assert.assertTrue(evalDouble("2.0 == 2.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("2.0 == 2.0", bindings));
Assert.assertTrue(evalDouble("2.0 != 1.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("2.0 != 1.0", bindings));
Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings)); Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings)); Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings)); Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings)); Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings)); Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings));
Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings)); Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings)); Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings));
Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings)); Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings)); Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings));
Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings)); Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings));
Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings)); Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings));
Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings)); Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings));
Assert.assertTrue(evalDouble("!-1.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("!-1.0", bindings));
Assert.assertTrue(evalDouble("!0.0", bindings) > 0.0); Assert.assertEquals(1L, evalLong("!0.0", bindings));
Assert.assertFalse(evalDouble("!2.0", bindings) > 0.0); Assert.assertEquals(0L, evalLong("!2.0", bindings));
}
finally {
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
Assert.assertEquals(0L, evalLong("1.0 && 0.0", bindings));
Assert.assertEquals(1L, evalLong("1.0 && 2.0", bindings));
Assert.assertEquals(1L, evalLong("1.0 || 0.0", bindings)); assertEquals(3.5, evalDouble("2.0 + 1.5", bindings), 0.0001);
Assert.assertEquals(0L, evalLong("0.0 || 0.0", bindings)); assertEquals(0.5, evalDouble("2.0 - 1.5", bindings), 0.0001);
assertEquals(3.0, evalDouble("2.0 * 1.5", bindings), 0.0001);
Assert.assertEquals(1L, evalLong("2.0 > 1.0", bindings)); assertEquals(4.0, evalDouble("2.0 / 0.5", bindings), 0.0001);
Assert.assertEquals(1L, evalLong("2.0 >= 2.0", bindings)); assertEquals(0.2, evalDouble("2.0 % 0.3", bindings), 0.0001);
Assert.assertEquals(1L, evalLong("1.0 < 2.0", bindings)); assertEquals(8.0, evalDouble("2.0 ^ 3.0", bindings), 0.0001);
Assert.assertEquals(1L, evalLong("2.0 <= 2.0", bindings)); assertEquals(-1.5, evalDouble("-1.5", bindings), 0.0001);
Assert.assertEquals(1L, evalLong("2.0 == 2.0", bindings));
Assert.assertEquals(1L, evalLong("2.0 != 1.0", bindings));
Assert.assertEquals(1L, evalLong("notdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(1L, evalLong("isdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("notdistinctfrom(2.0, 1.0)", bindings));
Assert.assertEquals(0L, evalLong("isdistinctfrom(2.0, 2.0)", bindings));
Assert.assertEquals(0L, evalLong("istrue(0.0)", bindings));
Assert.assertEquals(1L, evalLong("isfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("nottrue(0.0)", bindings));
Assert.assertEquals(0L, evalLong("notfalse(0.0)", bindings));
Assert.assertEquals(1L, evalLong("istrue(1.0)", bindings));
Assert.assertEquals(0L, evalLong("isfalse(1.0)", bindings));
Assert.assertEquals(0L, evalLong("nottrue(1.0)", bindings));
Assert.assertEquals(1L, evalLong("notfalse(1.0)", bindings));
Assert.assertEquals(1L, evalLong("!-1.0", bindings));
Assert.assertEquals(1L, evalLong("!0.0", bindings));
Assert.assertEquals(0L, evalLong("!2.0", bindings));
assertEquals(3.5, evalDouble("2.0 + 1.5", bindings), 0.0001);
assertEquals(0.5, evalDouble("2.0 - 1.5", bindings), 0.0001);
assertEquals(3.0, evalDouble("2.0 * 1.5", bindings), 0.0001);
assertEquals(4.0, evalDouble("2.0 / 0.5", bindings), 0.0001);
assertEquals(0.2, evalDouble("2.0 % 0.3", bindings), 0.0001);
assertEquals(8.0, evalDouble("2.0 ^ 3.0", bindings), 0.0001);
assertEquals(-1.5, evalDouble("-1.5", bindings), 0.0001);
assertEquals(2.0, evalDouble("sqrt(4.0)", bindings), 0.0001); assertEquals(2.0, evalDouble("sqrt(4.0)", bindings), 0.0001);
assertEquals(2.0, evalDouble("if(1.0, 2.0, 3.0)", bindings), 0.0001); assertEquals(2.0, evalDouble("if(1.0, 2.0, 3.0)", bindings), 0.0001);
assertEquals(3.0, evalDouble("if(0.0, 2.0, 3.0)", bindings), 0.0001); assertEquals(3.0, evalDouble("if(0.0, 2.0, 3.0)", bindings), 0.0001);
}
finally {
ExpressionProcessing.initializeForTests();
}
} }
@Test @Test
@ -934,56 +890,29 @@ public class EvalTest extends InitializedNullHandlingTest
ImmutableMap.of("x", 100L, "y", 100L, "z", 100D, "w", 100D) ImmutableMap.of("x", 100L, "y", 100L, "z", 100D, "w", 100D)
); );
try { ExprEval eval = Parser.parse("x==y", ExprMacroTable.nil()).eval(bindings);
ExpressionProcessing.initializeForStrictBooleansTests(false); Assert.assertTrue(eval.asBoolean());
ExprEval eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings); assertEquals(ExpressionType.LONG, eval.type());
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.DOUBLE, eval.type());
eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings); eval = Parser.parse("x!=y", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean()); Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.DOUBLE, eval.type()); assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings); eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean()); Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.DOUBLE, eval.type()); assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings); eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean()); Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.DOUBLE, eval.type()); assertEquals(ExpressionType.LONG, eval.type());
}
finally {
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(true);
ExprEval eval = Parser.parse("x==y", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("x!=y", ExprMacroTable.nil()).eval(bindings); eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean()); Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type()); assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("x==z", ExprMacroTable.nil()).eval(bindings); eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean()); Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type()); assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("x!=z", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z==w", ExprMacroTable.nil()).eval(bindings);
Assert.assertTrue(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
eval = Parser.parse("z!=w", ExprMacroTable.nil()).eval(bindings);
Assert.assertFalse(eval.asBoolean());
assertEquals(ExpressionType.LONG, eval.type());
}
finally {
ExpressionProcessing.initializeForTests();
}
} }
@Test @Test
@ -991,142 +920,60 @@ public class EvalTest extends InitializedNullHandlingTest
{ {
Expr.ObjectBinding bindings = InputBindings.nilBindings(); Expr.ObjectBinding bindings = InputBindings.nilBindings();
try { assertEquals(1L, eval("'true' && 'true'", bindings).value());
ExpressionProcessing.initializeForStrictBooleansTests(true); assertEquals(0L, eval("'true' && 'false'", bindings).value());
assertEquals(1L, eval("'true' && 'true'", bindings).value()); assertEquals(0L, eval("'false' && 'true'", bindings).value());
assertEquals(0L, eval("'true' && 'false'", bindings).value()); assertEquals(0L, eval("'troo' && 'true'", bindings).value());
assertEquals(0L, eval("'false' && 'true'", bindings).value()); assertEquals(0L, eval("'false' && 'false'", bindings).value());
assertEquals(0L, eval("'troo' && 'true'", bindings).value());
assertEquals(0L, eval("'false' && 'false'", bindings).value());
assertEquals(1L, eval("'true' || 'true'", bindings).value()); assertEquals(1L, eval("'true' || 'true'", bindings).value());
assertEquals(1L, eval("'true' || 'false'", bindings).value()); assertEquals(1L, eval("'true' || 'false'", bindings).value());
assertEquals(1L, eval("'false' || 'true'", bindings).value()); assertEquals(1L, eval("'false' || 'true'", bindings).value());
assertEquals(1L, eval("'troo' || 'true'", bindings).value()); assertEquals(1L, eval("'troo' || 'true'", bindings).value());
assertEquals(0L, eval("'false' || 'false'", bindings).value()); assertEquals(0L, eval("'false' || 'false'", bindings).value());
assertEquals(1L, eval("1 && 1", bindings).value()); assertEquals(1L, eval("1 && 1", bindings).value());
assertEquals(1L, eval("100 && 11", bindings).value()); assertEquals(1L, eval("100 && 11", bindings).value());
assertEquals(0L, eval("1 && 0", bindings).value()); assertEquals(0L, eval("1 && 0", bindings).value());
assertEquals(0L, eval("0 && 1", bindings).value()); assertEquals(0L, eval("0 && 1", bindings).value());
assertEquals(0L, eval("0 && 0", bindings).value()); assertEquals(0L, eval("0 && 0", bindings).value());
assertEquals(1L, eval("1 || 1", bindings).value()); assertEquals(1L, eval("1 || 1", bindings).value());
assertEquals(1L, eval("100 || 11", bindings).value()); assertEquals(1L, eval("100 || 11", bindings).value());
assertEquals(1L, eval("1 || 0", bindings).value()); assertEquals(1L, eval("1 || 0", bindings).value());
assertEquals(1L, eval("0 || 1", bindings).value()); assertEquals(1L, eval("0 || 1", bindings).value());
assertEquals(1L, eval("111 || 0", bindings).value()); assertEquals(1L, eval("111 || 0", bindings).value());
assertEquals(1L, eval("0 || 111", bindings).value()); assertEquals(1L, eval("0 || 111", bindings).value());
assertEquals(0L, eval("0 || 0", bindings).value()); assertEquals(0L, eval("0 || 0", bindings).value());
assertEquals(1L, eval("1.0 && 1.0", bindings).value()); assertEquals(1L, eval("1.0 && 1.0", bindings).value());
assertEquals(1L, eval("0.100 && 1.1", bindings).value()); assertEquals(1L, eval("0.100 && 1.1", bindings).value());
assertEquals(0L, eval("1.0 && 0.0", bindings).value()); assertEquals(0L, eval("1.0 && 0.0", bindings).value());
assertEquals(0L, eval("0.0 && 1.0", bindings).value()); assertEquals(0L, eval("0.0 && 1.0", bindings).value());
assertEquals(0L, eval("0.0 && 0.0", bindings).value()); assertEquals(0L, eval("0.0 && 0.0", bindings).value());
assertEquals(1L, eval("1.0 || 1.0", bindings).value()); assertEquals(1L, eval("1.0 || 1.0", bindings).value());
assertEquals(1L, eval("0.2 || 0.3", bindings).value()); assertEquals(1L, eval("0.2 || 0.3", bindings).value());
assertEquals(1L, eval("1.0 || 0.0", bindings).value()); assertEquals(1L, eval("1.0 || 0.0", bindings).value());
assertEquals(1L, eval("0.0 || 1.0", bindings).value()); assertEquals(1L, eval("0.0 || 1.0", bindings).value());
assertEquals(1L, eval("1.11 || 0.0", bindings).value()); assertEquals(1L, eval("1.11 || 0.0", bindings).value());
assertEquals(1L, eval("0.0 || 0.111", bindings).value()); assertEquals(1L, eval("0.0 || 0.111", bindings).value());
assertEquals(0L, eval("0.0 || 0.0", bindings).value()); assertEquals(0L, eval("0.0 || 0.0", bindings).value());
assertEquals(1L, eval("null || 1", bindings).value()); assertEquals(1L, eval("null || 1", bindings).value());
assertEquals(1L, eval("1 || null", bindings).value()); assertEquals(1L, eval("1 || null", bindings).value());
// in sql incompatible mode, null is false, so we return 0 // in sql incompatible mode, null is false, so we return 0
assertEquals(NullHandling.defaultLongValue(), eval("null || 0", bindings).valueOrDefault()); assertEquals(NullHandling.defaultLongValue(), eval("null || 0", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("0 || null", bindings).valueOrDefault()); assertEquals(NullHandling.defaultLongValue(), eval("0 || null", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("null || null", bindings).valueOrDefault()); assertEquals(NullHandling.defaultLongValue(), eval("null || null", bindings).valueOrDefault());
// in sql incompatible mode, null is false, so we return 0 // in sql incompatible mode, null is false, so we return 0
assertEquals(NullHandling.defaultLongValue(), eval("null && 1", bindings).valueOrDefault()); assertEquals(NullHandling.defaultLongValue(), eval("null && 1", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("1 && null", bindings).valueOrDefault()); assertEquals(NullHandling.defaultLongValue(), eval("1 && null", bindings).valueOrDefault());
assertEquals(NullHandling.defaultLongValue(), eval("null && null", bindings).valueOrDefault()); assertEquals(NullHandling.defaultLongValue(), eval("null && null", bindings).valueOrDefault());
// if either side is false, output is false in both modes // if either side is false, output is false in both modes
assertEquals(0L, eval("null && 0", bindings).value()); assertEquals(0L, eval("null && 0", bindings).value());
assertEquals(0L, eval("0 && null", bindings).value()); assertEquals(0L, eval("0 && null", bindings).value());
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
try {
// turn on legacy insanity mode
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertEquals("true", eval("'true' && 'true'", bindings).value());
assertEquals("false", eval("'true' && 'false'", bindings).value());
assertEquals("false", eval("'false' && 'true'", bindings).value());
assertEquals("troo", eval("'troo' && 'true'", bindings).value());
assertEquals("false", eval("'false' && 'false'", bindings).value());
assertEquals("true", eval("'true' || 'true'", bindings).value());
assertEquals("true", eval("'true' || 'false'", bindings).value());
assertEquals("true", eval("'false' || 'true'", bindings).value());
assertEquals("true", eval("'troo' || 'true'", bindings).value());
assertEquals("false", eval("'false' || 'false'", bindings).value());
assertEquals(1.0, eval("1.0 && 1.0", bindings).value());
assertEquals(1.1, eval("0.100 && 1.1", bindings).value());
assertEquals(0.0, eval("1.0 && 0.0", bindings).value());
assertEquals(0.0, eval("0.0 && 1.0", bindings).value());
assertEquals(0.0, eval("0.0 && 0.0", bindings).value());
assertEquals(1.0, eval("1.0 || 1.0", bindings).value());
assertEquals(0.2, eval("0.2 || 0.3", bindings).value());
assertEquals(1.0, eval("1.0 || 0.0", bindings).value());
assertEquals(1.0, eval("0.0 || 1.0", bindings).value());
assertEquals(1.11, eval("1.11 || 0.0", bindings).value());
assertEquals(0.111, eval("0.0 || 0.111", bindings).value());
assertEquals(0.0, eval("0.0 || 0.0", bindings).value());
assertEquals(1L, eval("1 && 1", bindings).value());
assertEquals(11L, eval("100 && 11", bindings).value());
assertEquals(0L, eval("1 && 0", bindings).value());
assertEquals(0L, eval("0 && 1", bindings).value());
assertEquals(0L, eval("0 && 0", bindings).value());
assertEquals(1L, eval("1 || 1", bindings).value());
assertEquals(100L, eval("100 || 11", bindings).value());
assertEquals(1L, eval("1 || 0", bindings).value());
assertEquals(1L, eval("0 || 1", bindings).value());
assertEquals(111L, eval("111 || 0", bindings).value());
assertEquals(111L, eval("0 || 111", bindings).value());
assertEquals(0L, eval("0 || 0", bindings).value());
assertEquals(1.0, eval("1.0 && 1.0", bindings).value());
assertEquals(1.1, eval("0.100 && 1.1", bindings).value());
assertEquals(0.0, eval("1.0 && 0.0", bindings).value());
assertEquals(0.0, eval("0.0 && 1.0", bindings).value());
assertEquals(0.0, eval("0.0 && 0.0", bindings).value());
assertEquals(1.0, eval("1.0 || 1.0", bindings).value());
assertEquals(0.2, eval("0.2 || 0.3", bindings).value());
assertEquals(1.0, eval("1.0 || 0.0", bindings).value());
assertEquals(1.0, eval("0.0 || 1.0", bindings).value());
assertEquals(1.11, eval("1.11 || 0.0", bindings).value());
assertEquals(0.111, eval("0.0 || 0.111", bindings).value());
assertEquals(0.0, eval("0.0 || 0.0", bindings).value());
assertEquals(1L, eval("null || 1", bindings).value());
assertEquals(1L, eval("1 || null", bindings).value());
assertEquals(0L, eval("null || 0", bindings).value());
Assert.assertNull(eval("0 || null", bindings).value());
Assert.assertNull(eval("null || null", bindings).value());
Assert.assertNull(eval("null && 1", bindings).value());
Assert.assertNull(eval("1 && null", bindings).value());
Assert.assertNull(eval("null && 0", bindings).value());
assertEquals(0L, eval("0 && null", bindings).value());
assertNull(eval("null && null", bindings).value());
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
} }
@Test @Test
@ -1143,86 +990,40 @@ public class EvalTest extends InitializedNullHandlingTest
bindingsMap.put("b2", false); bindingsMap.put("b2", false);
Expr.ObjectBinding bindings = InputBindings.forMap(bindingsMap); Expr.ObjectBinding bindings = InputBindings.forMap(bindingsMap);
try { assertEquals(1L, eval("s1 && s1", bindings).value());
ExpressionProcessing.initializeForStrictBooleansTests(true); assertEquals(0L, eval("s1 && s2", bindings).value());
assertEquals(1L, eval("s1 && s1", bindings).value()); assertEquals(0L, eval("s2 && s1", bindings).value());
assertEquals(0L, eval("s1 && s2", bindings).value()); assertEquals(0L, eval("s2 && s2", bindings).value());
assertEquals(0L, eval("s2 && s1", bindings).value());
assertEquals(0L, eval("s2 && s2", bindings).value());
assertEquals(1L, eval("s1 || s1", bindings).value()); assertEquals(1L, eval("s1 || s1", bindings).value());
assertEquals(1L, eval("s1 || s2", bindings).value()); assertEquals(1L, eval("s1 || s2", bindings).value());
assertEquals(1L, eval("s2 || s1", bindings).value()); assertEquals(1L, eval("s2 || s1", bindings).value());
assertEquals(0L, eval("s2 || s2", bindings).value()); assertEquals(0L, eval("s2 || s2", bindings).value());
assertEquals(1L, eval("l1 && l1", bindings).value()); assertEquals(1L, eval("l1 && l1", bindings).value());
assertEquals(0L, eval("l1 && l2", bindings).value()); assertEquals(0L, eval("l1 && l2", bindings).value());
assertEquals(0L, eval("l2 && l1", bindings).value()); assertEquals(0L, eval("l2 && l1", bindings).value());
assertEquals(0L, eval("l2 && l2", bindings).value()); assertEquals(0L, eval("l2 && l2", bindings).value());
assertEquals(1L, eval("b1 && b1", bindings).value()); assertEquals(1L, eval("b1 && b1", bindings).value());
assertEquals(0L, eval("b1 && b2", bindings).value()); assertEquals(0L, eval("b1 && b2", bindings).value());
assertEquals(0L, eval("b2 && b1", bindings).value()); assertEquals(0L, eval("b2 && b1", bindings).value());
assertEquals(0L, eval("b2 && b2", bindings).value()); assertEquals(0L, eval("b2 && b2", bindings).value());
assertEquals(1L, eval("d1 && d1", bindings).value()); assertEquals(1L, eval("d1 && d1", bindings).value());
assertEquals(0L, eval("d1 && d2", bindings).value()); assertEquals(0L, eval("d1 && d2", bindings).value());
assertEquals(0L, eval("d2 && d1", bindings).value()); assertEquals(0L, eval("d2 && d1", bindings).value());
assertEquals(0L, eval("d2 && d2", bindings).value()); assertEquals(0L, eval("d2 && d2", bindings).value());
assertEquals(1L, eval("b1", bindings).value()); assertEquals(1L, eval("b1", bindings).value());
assertEquals(1L, eval("if(b1,1,0)", bindings).value()); assertEquals(1L, eval("if(b1,1,0)", bindings).value());
assertEquals(1L, eval("if(l1,1,0)", bindings).value()); assertEquals(1L, eval("if(l1,1,0)", bindings).value());
assertEquals(1L, eval("if(d1,1,0)", bindings).value()); assertEquals(1L, eval("if(d1,1,0)", bindings).value());
assertEquals(1L, eval("if(s1,1,0)", bindings).value()); assertEquals(1L, eval("if(s1,1,0)", bindings).value());
assertEquals(0L, eval("if(b2,1,0)", bindings).value()); assertEquals(0L, eval("if(b2,1,0)", bindings).value());
assertEquals(0L, eval("if(l2,1,0)", bindings).value()); assertEquals(0L, eval("if(l2,1,0)", bindings).value());
assertEquals(0L, eval("if(d2,1,0)", bindings).value()); assertEquals(0L, eval("if(d2,1,0)", bindings).value());
assertEquals(0L, eval("if(s2,1,0)", bindings).value()); assertEquals(0L, eval("if(s2,1,0)", bindings).value());
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
try {
// turn on legacy insanity mode
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertEquals("true", eval("s1 && s1", bindings).value());
assertEquals("false", eval("s1 && s2", bindings).value());
assertEquals("false", eval("s2 && s1", bindings).value());
assertEquals("false", eval("s2 && s2", bindings).value());
assertEquals("true", eval("b1 && b1", bindings).value());
assertEquals("false", eval("b1 && b2", bindings).value());
assertEquals("false", eval("b2 && b1", bindings).value());
assertEquals("false", eval("b2 && b2", bindings).value());
assertEquals(100L, eval("l1 && l1", bindings).value());
assertEquals(0L, eval("l1 && l2", bindings).value());
assertEquals(0L, eval("l2 && l1", bindings).value());
assertEquals(0L, eval("l2 && l2", bindings).value());
assertEquals(1.1, eval("d1 && d1", bindings).value());
assertEquals(0.0, eval("d1 && d2", bindings).value());
assertEquals(0.0, eval("d2 && d1", bindings).value());
assertEquals(0.0, eval("d2 && d2", bindings).value());
assertEquals("true", eval("b1", bindings).value());
assertEquals(1L, eval("if(b1,1,0)", bindings).value());
assertEquals(1L, eval("if(l1,1,0)", bindings).value());
assertEquals(1L, eval("if(d1,1,0)", bindings).value());
assertEquals(1L, eval("if(s1,1,0)", bindings).value());
assertEquals(0L, eval("if(b2,1,0)", bindings).value());
assertEquals(0L, eval("if(l2,1,0)", bindings).value());
assertEquals(0L, eval("if(d2,1,0)", bindings).value());
assertEquals(0L, eval("if(s2,1,0)", bindings).value());
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
} }
@Test @Test
@ -1621,17 +1422,6 @@ public class EvalTest extends InitializedNullHandlingTest
assertBestEffortOf(true, ExpressionType.LONG, 1L); assertBestEffortOf(true, ExpressionType.LONG, 1L);
assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L}); assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L});
try {
// in non-strict boolean mode, they are strings
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertBestEffortOf(true, ExpressionType.STRING, "true");
assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"});
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
// doubles // doubles
assertBestEffortOf(1.0, ExpressionType.DOUBLE, 1.0); assertBestEffortOf(1.0, ExpressionType.DOUBLE, 1.0);
assertBestEffortOf(1.0f, ExpressionType.DOUBLE, 1.0); assertBestEffortOf(1.0f, ExpressionType.DOUBLE, 1.0);

106
processing/src/test/java/org/apache/druid/math/expr/OutputTypeTest.java
View File

@ -70,29 +70,12 @@ public class OutputTypeTest extends InitializedNullHandlingTest
assertOutputType("-y", inspector, ExpressionType.LONG); assertOutputType("-y", inspector, ExpressionType.LONG);
assertOutputType("-z", inspector, ExpressionType.DOUBLE); assertOutputType("-z", inspector, ExpressionType.DOUBLE);
try { assertOutputType("!'true'", inspector, ExpressionType.LONG);
ExpressionProcessing.initializeForStrictBooleansTests(true); assertOutputType("!1", inspector, ExpressionType.LONG);
assertOutputType("!'true'", inspector, ExpressionType.LONG); assertOutputType("!x", inspector, ExpressionType.LONG);
assertOutputType("!1", inspector, ExpressionType.LONG); assertOutputType("!y", inspector, ExpressionType.LONG);
assertOutputType("!x", inspector, ExpressionType.LONG); assertOutputType("!1.1", inspector, ExpressionType.LONG);
assertOutputType("!y", inspector, ExpressionType.LONG); assertOutputType("!z", inspector, ExpressionType.LONG);
assertOutputType("!1.1", inspector, ExpressionType.LONG);
assertOutputType("!z", inspector, ExpressionType.LONG);
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertOutputType("!1.1", inspector, ExpressionType.DOUBLE);
assertOutputType("!z", inspector, ExpressionType.DOUBLE);
}
finally {
// reset
ExpressionProcessing.initializeForTests();
}
} }
@Test @Test
@ -126,61 +109,32 @@ public class OutputTypeTest extends InitializedNullHandlingTest
assertOutputType("z^z_", inspector, ExpressionType.DOUBLE); assertOutputType("z^z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z%z_", inspector, ExpressionType.DOUBLE); assertOutputType("z%z_", inspector, ExpressionType.DOUBLE);
try { assertOutputType("y>y_", inspector, ExpressionType.LONG);
ExpressionProcessing.initializeForStrictBooleansTests(true); assertOutputType("y_<y", inspector, ExpressionType.LONG);
assertOutputType("y>y_", inspector, ExpressionType.LONG); assertOutputType("y_<=y", inspector, ExpressionType.LONG);
assertOutputType("y_<y", inspector, ExpressionType.LONG); assertOutputType("y_>=y", inspector, ExpressionType.LONG);
assertOutputType("y_<=y", inspector, ExpressionType.LONG); assertOutputType("y_==y", inspector, ExpressionType.LONG);
assertOutputType("y_>=y", inspector, ExpressionType.LONG); assertOutputType("y_!=y", inspector, ExpressionType.LONG);
assertOutputType("y_==y", inspector, ExpressionType.LONG); assertOutputType("y_ && y", inspector, ExpressionType.LONG);
assertOutputType("y_!=y", inspector, ExpressionType.LONG); assertOutputType("y_ || y", inspector, ExpressionType.LONG);
assertOutputType("y_ && y", inspector, ExpressionType.LONG);
assertOutputType("y_ || y", inspector, ExpressionType.LONG);
assertOutputType("z>y_", inspector, ExpressionType.LONG); assertOutputType("z>y_", inspector, ExpressionType.LONG);
assertOutputType("z<y", inspector, ExpressionType.LONG); assertOutputType("z<y", inspector, ExpressionType.LONG);
assertOutputType("z<=y", inspector, ExpressionType.LONG); assertOutputType("z<=y", inspector, ExpressionType.LONG);
assertOutputType("y>=z", inspector, ExpressionType.LONG); assertOutputType("y>=z", inspector, ExpressionType.LONG);
assertOutputType("z==y", inspector, ExpressionType.LONG); assertOutputType("z==y", inspector, ExpressionType.LONG);
assertOutputType("z!=y", inspector, ExpressionType.LONG); assertOutputType("z!=y", inspector, ExpressionType.LONG);
assertOutputType("z && y", inspector, ExpressionType.LONG); assertOutputType("z && y", inspector, ExpressionType.LONG);
assertOutputType("y || z", inspector, ExpressionType.LONG); assertOutputType("y || z", inspector, ExpressionType.LONG);
assertOutputType("z>z_", inspector, ExpressionType.LONG); assertOutputType("z>z_", inspector, ExpressionType.LONG);
assertOutputType("z<z_", inspector, ExpressionType.LONG); assertOutputType("z<z_", inspector, ExpressionType.LONG);
assertOutputType("z<=z_", inspector, ExpressionType.LONG); assertOutputType("z<=z_", inspector, ExpressionType.LONG);
assertOutputType("z_>=z", inspector, ExpressionType.LONG); assertOutputType("z_>=z", inspector, ExpressionType.LONG);
assertOutputType("z==z_", inspector, ExpressionType.LONG); assertOutputType("z==z_", inspector, ExpressionType.LONG);
assertOutputType("z!=z_", inspector, ExpressionType.LONG); assertOutputType("z!=z_", inspector, ExpressionType.LONG);
assertOutputType("z && z_", inspector, ExpressionType.LONG); assertOutputType("z && z_", inspector, ExpressionType.LONG);
assertOutputType("z_ || z", inspector, ExpressionType.LONG); assertOutputType("z_ || z", inspector, ExpressionType.LONG);
}
finally {
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
assertOutputType("z>y_", inspector, ExpressionType.DOUBLE);
assertOutputType("z<y", inspector, ExpressionType.DOUBLE);
assertOutputType("z<=y", inspector, ExpressionType.DOUBLE);
assertOutputType("y>=z", inspector, ExpressionType.DOUBLE);
assertOutputType("z==y", inspector, ExpressionType.DOUBLE);
assertOutputType("z!=y", inspector, ExpressionType.DOUBLE);
assertOutputType("z && y", inspector, ExpressionType.DOUBLE);
assertOutputType("y || z", inspector, ExpressionType.DOUBLE);
assertOutputType("z>z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z<z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z<=z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z_>=z", inspector, ExpressionType.DOUBLE);
assertOutputType("z==z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z!=z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z && z_", inspector, ExpressionType.DOUBLE);
assertOutputType("z_ || z", inspector, ExpressionType.DOUBLE);
}
finally {
ExpressionProcessing.initializeForTests();
}
assertOutputType("1*(2 + 3.0)", inspector, ExpressionType.DOUBLE); assertOutputType("1*(2 + 3.0)", inspector, ExpressionType.DOUBLE);
} }

71
processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
View File

@ -30,7 +30,6 @@ import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequence;
import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.Druids; import org.apache.druid.query.Druids;
import org.apache.druid.query.NestedDataTestUtils; import org.apache.druid.query.NestedDataTestUtils;
import org.apache.druid.query.Query; import org.apache.druid.query.Query;
@ -668,76 +667,6 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString()); Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
} }
@Test
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesNonStrictBooleans() throws Exception
{
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
.dataSource("test_datasource")
.intervals(
new MultipleIntervalSegmentSpec(
Collections.singletonList(Intervals.ETERNITY)
)
)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.limit(100)
.context(ImmutableMap.of());
Query<ScanResultValue> scanQuery = builder.build();
final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")};
List<Segment> realtimeSegs = ImmutableList.of(
NestedDataTestUtils.createIncrementalIndex(
tempFolder,
NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
TestIndex.DEFAULT_JSON_INPUT_FORMAT,
NestedDataTestUtils.TIMESTAMP_SPEC,
NestedDataTestUtils.AUTO_DISCOVERY,
TransformSpec.NONE,
aggs,
Granularities.NONE,
true
)
);
List<Segment> segs = NestedDataTestUtils.createSegments(
tempFolder,
closer,
NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
TestIndex.DEFAULT_JSON_INPUT_FORMAT,
NestedDataTestUtils.TIMESTAMP_SPEC,
NestedDataTestUtils.AUTO_DISCOVERY,
TransformSpec.NONE,
aggs,
Granularities.NONE,
true,
IndexSpec.DEFAULT
);
final Sequence<ScanResultValue> seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery);
final Sequence<ScanResultValue> seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery);
List<ScanResultValue> resultsRealtime = seq.toList();
List<ScanResultValue> resultsSegments = seq2.toList();
logResults(resultsSegments);
logResults(resultsRealtime);
Assert.assertEquals(1, resultsRealtime.size());
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
Assert.assertEquals(
"["
+ "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
resultsSegments.get(0).getEvents().toString()
);
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
}
finally {
ExpressionProcessing.initializeForTests();
}
}
@Test @Test
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception
{ {

143
processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java
View File

@ -1,143 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.segment.filter;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.filter.NotDimFilter;
import org.apache.druid.segment.CursorFactory;
import org.apache.druid.segment.IndexBuilder;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.Closeable;
@RunWith(Parameterized.class)
public class ExpressionFilterNonStrictBooleansTest extends ExpressionFilterTest
{
public ExpressionFilterNonStrictBooleansTest(
String testName,
IndexBuilder indexBuilder,
Function<IndexBuilder, Pair<CursorFactory, Closeable>> finisher,
boolean cnf,
boolean optimize
)
{
super(testName, indexBuilder, finisher, cnf, optimize);
}
@Before
@Override
public void setup()
{
ExpressionProcessing.initializeForStrictBooleansTests(false);
}
@Override
@Test
public void testComplement()
{
if (NullHandling.sqlCompatible()) {
assertFilterMatches(edf("dim5 == 'a'"), ImmutableList.of("0"));
// non-strict mode is wild
assertFilterMatches(
NotDimFilter.of(edf("dim5 == 'a'")),
ImmutableList.of("1", "2", "3", "4", "5", "6", "7", "8", "9")
);
assertFilterMatches(
edf("dim5 == ''"), ImmutableList.of("4")
);
// non-strict mode!
assertFilterMatches(
NotDimFilter.of(edf("dim5 == ''")), ImmutableList.of("0", "1", "2", "3", "5", "6", "7", "8", "9")
);
} else {
assertFilterMatches(edf("dim5 == 'a'"), ImmutableList.of("0"));
assertFilterMatches(
NotDimFilter.of(edf("dim5 == 'a'")),
ImmutableList.of("1", "2", "3", "4", "5", "6", "7", "8", "9")
);
}
}
@Override
@Test
public void testMissingColumn()
{
if (NullHandling.replaceWithDefault()) {
assertFilterMatches(
edf("missing == ''"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
assertFilterMatches(
edf("missing == otherMissing"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
} else {
// AS per SQL standard null == null returns false.
assertFilterMatches(edf("missing == null"), ImmutableList.of());
// in non-strict mode, madness happens
assertFilterMatches(
NotDimFilter.of(edf("missing == null")),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
// also this madness doesn't do madness
assertFilterMatches(
edf("missing == otherMissing"),
ImmutableList.of()
);
assertFilterMatches(
NotDimFilter.of(edf("missing == otherMissing")),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
}
assertFilterMatches(edf("missing == '1'"), ImmutableList.of());
assertFilterMatches(edf("missing == 2"), ImmutableList.of());
if (NullHandling.replaceWithDefault()) {
// missing equivaluent to 0
assertFilterMatches(
edf("missing < '2'"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
assertFilterMatches(
edf("missing < 2"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
assertFilterMatches(
edf("missing < 2.0"),
ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
);
} else {
// missing equivalent to null
assertFilterMatches(edf("missing < '2'"), ImmutableList.of());
assertFilterMatches(edf("missing < 2"), ImmutableList.of());
assertFilterMatches(edf("missing < 2.0"), ImmutableList.of());
}
assertFilterMatches(edf("missing > '2'"), ImmutableList.of());
assertFilterMatches(edf("missing > 2"), ImmutableList.of());
assertFilterMatches(edf("missing > 2.0"), ImmutableList.of());
assertFilterMatchesSkipVectorize(edf("like(missing, '1%')"), ImmutableList.of());
}
}

17
processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
View File

@ -36,7 +36,6 @@ import org.apache.druid.data.input.impl.TimeAndDimsParseSpec;
import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.Pair;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.filter.ExpressionDimFilter; import org.apache.druid.query.filter.ExpressionDimFilter;
import org.apache.druid.query.filter.Filter; import org.apache.druid.query.filter.Filter;
@ -46,10 +45,8 @@ import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.junit.runners.Parameterized; import org.junit.runners.Parameterized;
@ -126,22 +123,10 @@ public class ExpressionFilterTest extends BaseFilterTest
); );
} }
@Before
public void setup()
{
ExpressionProcessing.initializeForStrictBooleansTests(true);
}
@After
public void teardown()
{
ExpressionProcessing.initializeForTests();
}
@AfterClass @AfterClass
public static void tearDown() throws Exception public static void tearDown() throws Exception
{ {
BaseFilterTest.tearDown(ColumnComparisonFilterTest.class.getName()); BaseFilterTest.tearDown(ExpressionFilterTest.class.getName());
} }
@Test @Test

78
processing/src/test/java/org/apache/druid/segment/transform/TransformSpecTest.java
View File

@ -30,7 +30,6 @@ import org.apache.druid.data.input.impl.MapInputRowParser;
import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; import org.apache.druid.data.input.impl.TimeAndDimsParseSpec;
import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.filter.AndDimFilter; import org.apache.druid.query.filter.AndDimFilter;
import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.query.filter.SelectorDimFilter;
@ -208,68 +207,31 @@ public class TransformSpecTest extends InitializedNullHandlingTest
@Test @Test
public void testBoolTransforms() public void testBoolTransforms()
{ {
try { final TransformSpec transformSpec = new TransformSpec(
ExpressionProcessing.initializeForStrictBooleansTests(true); null,
final TransformSpec transformSpec = new TransformSpec( ImmutableList.of(
null, new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE),
ImmutableList.of( new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE)
new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE), )
new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE) );
)
);
Assert.assertEquals( Assert.assertEquals(
ImmutableSet.of("bool"), ImmutableSet.of("bool"),
transformSpec.getRequiredColumns() transformSpec.getRequiredColumns()
); );
final InputRowParser<Map<String, Object>> parser = transformSpec.decorate(PARSER); final InputRowParser<Map<String, Object>> parser = transformSpec.decorate(PARSER);
final InputRow row = parser.parseBatch(ROW1).get(0); final InputRow row = parser.parseBatch(ROW1).get(0);
Assert.assertNotNull(row); Assert.assertNotNull(row);
Assert.assertEquals(1L, row.getRaw("truthy1")); Assert.assertEquals(1L, row.getRaw("truthy1"));
Assert.assertEquals(1L, row.getRaw("truthy2")); Assert.assertEquals(1L, row.getRaw("truthy2"));
final InputRow row2 = parser.parseBatch(ROW2).get(0); final InputRow row2 = parser.parseBatch(ROW2).get(0);
Assert.assertNotNull(row2); Assert.assertNotNull(row2);
Assert.assertEquals(0L, row2.getRaw("truthy1")); Assert.assertEquals(0L, row2.getRaw("truthy1"));
Assert.assertEquals(0L, row2.getRaw("truthy2")); Assert.assertEquals(0L, row2.getRaw("truthy2"));
}
finally {
ExpressionProcessing.initializeForTests();
}
try {
ExpressionProcessing.initializeForStrictBooleansTests(false);
final TransformSpec transformSpec = new TransformSpec(
null,
ImmutableList.of(
new ExpressionTransform("truthy1", "bool", TestExprMacroTable.INSTANCE),
new ExpressionTransform("truthy2", "if(bool,1,0)", TestExprMacroTable.INSTANCE)
)
);
Assert.assertEquals(
ImmutableSet.of("bool"),
transformSpec.getRequiredColumns()
);
final InputRowParser<Map<String, Object>> parser = transformSpec.decorate(PARSER);
final InputRow row = parser.parseBatch(ROW1).get(0);
Assert.assertNotNull(row);
Assert.assertEquals("true", row.getRaw("truthy1"));
Assert.assertEquals(1L, row.getRaw("truthy2"));
final InputRow row2 = parser.parseBatch(ROW2).get(0);
Assert.assertNotNull(row2);
Assert.assertEquals("false", row2.getRaw("truthy1"));
Assert.assertEquals(0L, row2.getRaw("truthy2"));
}
finally {
ExpressionProcessing.initializeForTests();
}
} }
@Test @Test

3
sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRule.java
View File

@ -27,7 +27,6 @@ import org.apache.calcite.rel.logical.LogicalValues;
import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLiteral;
import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.config.NullHandling;
import org.apache.druid.error.InvalidSqlInput; import org.apache.druid.error.InvalidSqlInput;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.InlineDataSource; import org.apache.druid.query.InlineDataSource;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.Calcites;
@ -122,7 +121,7 @@ public class DruidLogicalValuesRule extends RelOptRule
} }
return ((Number) RexLiteral.value(literal)).longValue(); return ((Number) RexLiteral.value(literal)).longValue();
case BOOLEAN: case BOOLEAN:
if (ExpressionProcessing.useStrictBooleans() && NullHandling.sqlCompatible() && literal.isNull()) { if (NullHandling.sqlCompatible() && literal.isNull()) {
return null; return null;
} }
return literal.isAlwaysTrue() ? 1L : 0L; return literal.isAlwaysTrue() ? 1L : 0L;

3
sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java
View File

@ -31,7 +31,6 @@ import org.apache.calcite.util.TimestampString;
import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.config.NullHandling;
import org.apache.druid.error.DruidExceptionMatcher; import org.apache.druid.error.DruidExceptionMatcher;
import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.sql.calcite.planner.DruidTypeSystem; import org.apache.druid.sql.calcite.planner.DruidTypeSystem;
import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.testing.InitializedNullHandlingTest;
@ -144,7 +143,7 @@ public class DruidLogicalValuesRuleTest
{ {
RexLiteral literal = REX_BUILDER.makeLiteral(null, REX_BUILDER.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN)); RexLiteral literal = REX_BUILDER.makeLiteral(null, REX_BUILDER.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN));
if (NullHandling.sqlCompatible() && ExpressionProcessing.useStrictBooleans()) { if (NullHandling.sqlCompatible()) {
final Object fromLiteral = DruidLogicalValuesRule.getValueFromLiteral(literal, DEFAULT_CONTEXT); final Object fromLiteral = DruidLogicalValuesRule.getValueFromLiteral(literal, DEFAULT_CONTEXT);
Assert.assertNull(fromLiteral); Assert.assertNull(fromLiteral);
} else { } else {