From 194a9c9abc1b6edf98c2652687515200c6cbd3e8 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 22 Aug 2023 00:19:56 -0700 Subject: [PATCH] set druid.expressions.useStrictBooleans to true by default (#14734) --- .../query/SqlExpressionBenchmark.java | 2 +- .../query/SqlNestedDataBenchmark.java | 2 +- docs/configuration/index.md | 4 +- docs/development/experimental-features.md | 1 - docs/ingestion/schema-design.md | 4 +- docs/querying/math-expr.md | 24 ++- docs/querying/sql-data-types.md | 6 +- .../InputSourceSamplerDiscoveryTest.java | 129 +++++++-------- .../org/apache/druid/math/expr/ExprEval.java | 3 - .../math/expr/ExpressionProcessingConfig.java | 13 +- .../org/apache/druid/math/expr/EvalTest.java | 15 +- .../druid/math/expr/VectorExprSanityTest.java | 14 -- .../VectorExpressionsSanityTest.java | 15 -- .../query/scan/NestedDataScanQueryTest.java | 153 +++++++++--------- ...xpressionFilterNonStrictBooleansTest.java} | 6 +- .../segment/filter/ExpressionFilterTest.java | 2 +- .../incremental/IncrementalIndexTest.java | 2 +- .../segment/transform/TransformerTest.java | 3 +- .../calcite/CalciteNestedDataQueryTest.java | 54 +++---- .../SqlVectorizedExpressionSanityTest.java | 3 - 20 files changed, 215 insertions(+), 240 deletions(-) rename processing/src/test/java/org/apache/druid/segment/filter/{ExpressionFilterStrictBooleansTest.java => ExpressionFilterNonStrictBooleansTest.java} (88%) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 80d208c5a43..6f96a2cde56 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -84,7 +84,7 @@ public class SqlExpressionBenchmark static { NullHandling.initializeForTests(); - ExpressionProcessing.initializeForStrictBooleansTests(true); + ExpressionProcessing.initializeForTests(); } private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig() diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java index da42aaeefcd..fe5dfff0c84 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java @@ -90,7 +90,7 @@ public class SqlNestedDataBenchmark static { NullHandling.initializeForTests(); - ExpressionProcessing.initializeForStrictBooleansTests(true); + ExpressionProcessing.initializeForTests(); } private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig() diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 362e2d553b3..cb4ee2edee4 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -2288,8 +2288,8 @@ Supported query contexts: |Key|Description|Default| |---|-----------|-------| -|`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean operators and functions, if set to `true` all boolean values will be either a `1` or `0`. See [expression documentation](../querying/math-expr.md#logical-operator-modes)|false| -|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can create nested arrays.|false| +|`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean operators and functions, if set to `true` all boolean values are either `1` or `0`. See [expression documentation](../querying/math-expr.md#logical-operator-modes) for more information.|true| +|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can create nested arrays.|true| ### Router #### Router Process Configs diff --git a/docs/development/experimental-features.md b/docs/development/experimental-features.md index d33f634a4b6..36c72822b4d 100644 --- a/docs/development/experimental-features.md +++ b/docs/development/experimental-features.md @@ -57,4 +57,3 @@ Note that this document does not track the status of contrib extensions, all of - [Configuration reference](../configuration/index.md) - `CLOSED_SEGMENTS_SINKS` mode - - Expression processing configuration `druid.expressions.allowNestedArrays` diff --git a/docs/ingestion/schema-design.md b/docs/ingestion/schema-design.md index 556cdc41a4b..0749d20b695 100644 --- a/docs/ingestion/schema-design.md +++ b/docs/ingestion/schema-design.md @@ -258,8 +258,8 @@ You can have Druid infer the schema and types for your data partially or fully b When performing type-aware schema discovery, Druid can discover all of the columns of your input data (that aren't in the exclusion list). Druid automatically chooses the most appropriate native Druid type among `STRING`, `LONG`, `DOUBLE`, `ARRAY`, `ARRAY`, `ARRAY`, or `COMPLEX` for nested data. For input formats with -native boolean types, Druid ingests these values as strings if `druid.expressions.useStrictBooleans` is set to `false` -(the default), or longs if set to `true` (for more SQL compatible behavior). Array typed columns can be queried using +native boolean types, Druid ingests these values as longs if `druid.expressions.useStrictBooleans` is set to `true` +(the default) or strings if set to `false`. Array typed columns can be queried using the [array functions](../querying/sql-array-functions.md) or [UNNEST](../querying/sql-functions.md#unnest). Nested columns can be queried with the [JSON functions](../querying/sql-json-functions.md). diff --git a/docs/querying/math-expr.md b/docs/querying/math-expr.md index 3da1fd39819..dcb4b7ce778 100644 --- a/docs/querying/math-expr.md +++ b/docs/querying/math-expr.md @@ -299,7 +299,7 @@ Supported features: * constants and identifiers are supported for any column type * `cast` is supported for numeric and string types * math operators: `+`,`-`,`*`,`/`,`%`,`^` are supported for numeric types -* logical operators: `!`, `&&`, `||`, are supported for string and numeric types (if `druid.expressions.useStrictBooleans=true`) +* logical operators: `!`, `&&`, `||`, are supported for string and numeric types * comparison operators: `=`, `!=`, `>`, `>=`, `<`, `<=` are supported for string and numeric types * math functions: `abs`, `acos`, `asin`, `atan`, `cbrt`, `ceil`, `cos`, `cosh`, `cot`, `exp`, `expm1`, `floor`, `getExponent`, `log`, `log10`, `log1p`, `nextUp`, `rint`, `signum`, `sin`, `sinh`, `sqrt`, `tan`, `tanh`, `toDegrees`, `toRadians`, `ulp`, `atan2`, `copySign`, `div`, `hypot`, `max`, `min`, `nextAfter`, `pow`, `remainder`, `scalb` are supported for numeric types * time functions: `timestamp_floor` (with constant granularity argument) is supported for numeric types @@ -309,9 +309,7 @@ Supported features: * other: `parse_long` is supported for numeric and string types ## Logical operator modes -Prior to the 0.23 release of Apache Druid, boolean function expressions have inconsistent handling of true and false values, and the logical 'and' and 'or' operators behave in a manner that is incompatible with SQL, even if SQL compatible null handling mode (`druid.generic.useDefaultValueForNull=false`) is enabled. Logical operators also pass through their input values similar to many scripting languages, and treat `null` as false, which can result in some rather strange behavior. Other boolean operations, such as comparisons and equality, retain their input types (e.g. `DOUBLE` comparison would produce `1.0` for true and `0.0` for false), while many other boolean functions strictly produce `LONG` typed values of `1` for true and `0` for false. - -After 0.23, while the inconsistent legacy behavior is still the default, it can be optionally be changed by setting `druid.expressions.useStrictBooleans=true`, so that these operations will allow correctly treating `null` values as "unknown" for SQL compatible behavior, and _all boolean output functions_ will output 'homogeneous' `LONG` typed boolean values of `1` for `true` and `0` for `false`. Additionally, +In Druid 28.0 and later, `druid.expressions.useStrictBooleans=true` is set by default. Logical operations treat `null` values as "unknown" for SQL compatible behavior. _All boolean output functions_ will output 'homogeneous' `LONG` typed boolean values of `1` for `true` and `0` for `false`. For the "or" operator: * `true || null`, `null || true`, -> `1` @@ -322,15 +320,8 @@ For the "and" operator: * `false && null`, `null && false` -> `0` Druid currently still retains implicit conversion of `LONG`, `DOUBLE`, and `STRING` types into boolean values in both modes: -* `LONG` or `DOUBLE` - any value greater than 0 is considered `true`, else `false` -* `STRING` - the value `'true'` (case insensitive) is considered `true`, everything else is `false`. - -Legacy behavior: -* `100 && 11` -> `11` -* `0.7 || 0.3` -> `0.3` -* `100 && 0` -> `0` -* `'troo' && 'true'` -> `'troo'` -* `'troo' || 'true'` -> `'true'` +* `LONG` or `DOUBLE`: any value greater than 0 is considered `true`, else `false`. +* `STRING`: the value `'true'` (case insensitive) is considered `true`, everything else is `false`. SQL compatible behavior: * `100 && 11` -> `1` @@ -339,4 +330,11 @@ SQL compatible behavior: * `'troo' && 'true'` -> `0` * `'troo' || 'true'` -> `1` +Prior to Druid 28.0.0, `druid.expressions.useStrictBooleans=false` was the default. In this mode, boolean function expressions have inconsistent handling of true and false values. The logical 'and' and 'or' operators behave in a manner that is incompatible with SQL, even if SQL compatible null handling mode (`druid.generic.useDefaultValueForNull=false`) is enabled. Logical operators also pass through their input values, similar to many scripting languages, and treat `null` as false, which results in some rather strange behavior. Other boolean operations, such as comparisons and equality, retain their input types (e.g. `DOUBLE` comparison produces `1.0` for true and `0.0` for false), while many other boolean functions strictly produce `LONG` typed values of `1` for true and `0` for false. This legacy mode can still be enabled by setting `druid.expressions.useStrictBooleans=false`. +Legacy behavior: +* `100 && 11` -> `11` +* `0.7 || 0.3` -> `0.3` +* `100 && 0` -> `0` +* `'troo' && 'true'` -> `'troo'` +* `'troo' || 'true'` -> `'true'` diff --git a/docs/querying/sql-data-types.md b/docs/querying/sql-data-types.md index 6fb5cc0764e..6f01369e844 100644 --- a/docs/querying/sql-data-types.md +++ b/docs/querying/sql-data-types.md @@ -153,13 +153,13 @@ values are treated as zeroes. This was the default prior to Druid 28.0.0. ## Boolean logic The [`druid.expressions.useStrictBooleans`](../configuration/index.md#expression-processing-configurations) -runtime property controls Druid's boolean logic mode. For the most SQL compliant behavior, set this to `true`. +runtime property controls Druid's boolean logic mode. For the most SQL compliant behavior, set this to `true` (the default). -When `druid.expressions.useStrictBooleans = false` (the default mode), Druid uses three-valued logic for +When `druid.expressions.useStrictBooleans = true`, Druid uses three-valued logic for [expressions](math-expr.md) evaluation, such as `expression` virtual columns or `expression` filters. However, even in this mode, Druid uses two-valued logic for filter types other than `expression`. -When `druid.expressions.useStrictBooleans = true` (legacy mode), Druid uses two-valued logic. +When `druid.expressions.useStrictBooleans = false` (legacy mode), Druid uses two-valued logic. ## Nested columns diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java index b510b6acb20..52fec4c80ce 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java @@ -58,71 +58,11 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest private InputSourceSampler inputSourceSampler = new InputSourceSampler(OBJECT_MAPPER); @Test - public void testDiscoveredTypes() + public void testDiscoveredTypesNonStrictBooleans() { - final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n')); - final SamplerResponse response = inputSourceSampler.sample( - inputSource, - new JsonInputFormat(null, null, null, null, null), - new DataSchema( - "test", - new TimestampSpec("t", null, null), - DimensionsSpec.builder().useSchemaDiscovery(true).build(), - null, - null, - null - ), - null - ); - Assert.assertEquals(6, response.getNumRowsRead()); - Assert.assertEquals(5, response.getNumRowsIndexed()); - Assert.assertEquals(6, response.getData().size()); - Assert.assertEquals( - ImmutableList.of( - new StringDimensionSchema("string"), - new LongDimensionSchema("long"), - new DoubleDimensionSchema("double"), - new StringDimensionSchema("bool"), - new StringDimensionSchema("variant"), - new AutoTypeColumnSchema("array"), - new AutoTypeColumnSchema("nested") - ), - response.getLogicalDimensions() - ); - - Assert.assertEquals( - ImmutableList.of( - new AutoTypeColumnSchema("string"), - new AutoTypeColumnSchema("long"), - new AutoTypeColumnSchema("double"), - new AutoTypeColumnSchema("bool"), - new AutoTypeColumnSchema("variant"), - new AutoTypeColumnSchema("array"), - new AutoTypeColumnSchema("nested") - ), - response.getPhysicalDimensions() - ); - Assert.assertEquals( - RowSignature.builder() - .addTimeColumn() - .add("string", ColumnType.STRING) - .add("long", ColumnType.LONG) - .add("double", ColumnType.DOUBLE) - .add("bool", ColumnType.STRING) - .add("variant", ColumnType.STRING) - .add("array", ColumnType.LONG_ARRAY) - .add("nested", ColumnType.NESTED_DATA) - .build(), - response.getLogicalSegmentSchema() - ); - } - - @Test - public void testDiscoveredTypesStrictBooleans() - { try { - ExpressionProcessing.initializeForStrictBooleansTests(true); + ExpressionProcessing.initializeForStrictBooleansTests(false); final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n')); final SamplerResponse response = inputSourceSampler.sample( inputSource, @@ -146,7 +86,7 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest new StringDimensionSchema("string"), new LongDimensionSchema("long"), new DoubleDimensionSchema("double"), - new LongDimensionSchema("bool"), + new StringDimensionSchema("bool"), new StringDimensionSchema("variant"), new AutoTypeColumnSchema("array"), new AutoTypeColumnSchema("nested") @@ -172,7 +112,7 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest .add("string", ColumnType.STRING) .add("long", ColumnType.LONG) .add("double", ColumnType.DOUBLE) - .add("bool", ColumnType.LONG) + .add("bool", ColumnType.STRING) .add("variant", ColumnType.STRING) .add("array", ColumnType.LONG_ARRAY) .add("nested", ColumnType.NESTED_DATA) @@ -185,6 +125,67 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest } } + @Test + public void testDiscoveredTypesStrictBooleans() + { + final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n')); + final SamplerResponse response = inputSourceSampler.sample( + inputSource, + new JsonInputFormat(null, null, null, null, null), + new DataSchema( + "test", + new TimestampSpec("t", null, null), + DimensionsSpec.builder().useSchemaDiscovery(true).build(), + null, + null, + null + ), + null + ); + + Assert.assertEquals(6, response.getNumRowsRead()); + Assert.assertEquals(5, response.getNumRowsIndexed()); + Assert.assertEquals(6, response.getData().size()); + Assert.assertEquals( + ImmutableList.of( + new StringDimensionSchema("string"), + new LongDimensionSchema("long"), + new DoubleDimensionSchema("double"), + new LongDimensionSchema("bool"), + new StringDimensionSchema("variant"), + new AutoTypeColumnSchema("array"), + new AutoTypeColumnSchema("nested") + ), + response.getLogicalDimensions() + ); + + Assert.assertEquals( + ImmutableList.of( + new AutoTypeColumnSchema("string"), + new AutoTypeColumnSchema("long"), + new AutoTypeColumnSchema("double"), + new AutoTypeColumnSchema("bool"), + new AutoTypeColumnSchema("variant"), + new AutoTypeColumnSchema("array"), + new AutoTypeColumnSchema("nested") + ), + response.getPhysicalDimensions() + ); + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("string", ColumnType.STRING) + .add("long", ColumnType.LONG) + .add("double", ColumnType.DOUBLE) + .add("bool", ColumnType.LONG) + .add("variant", ColumnType.STRING) + .add("array", ColumnType.LONG_ARRAY) + .add("nested", ColumnType.NESTED_DATA) + .build(), + response.getLogicalSegmentSchema() + ); + } + @Test public void testTypesClassicDiscovery() { diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java index 4bd29a19d0d..af1faf9eaf5 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -561,9 +561,6 @@ public abstract class ExprEval return ofDouble((Number) value); } if (value instanceof Boolean) { - if (ExpressionProcessing.useStrictBooleans()) { - return ofLongBoolean((Boolean) value); - } return ofDouble(Evals.asDouble((Boolean) value)); } if (value instanceof String) { diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java index 44e3b9409d6..2f12bd36d7a 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java @@ -50,7 +50,11 @@ public class ExpressionProcessingConfig @JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays ) { - this.useStrictBooleans = getWithPropertyFallbackFalse(useStrictBooleans, NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING); + this.useStrictBooleans = getWithPropertyFallback( + useStrictBooleans, + NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING, + "true" + ); this.processArraysAsMultiValueStrings = getWithPropertyFallbackFalse( processArraysAsMultiValueStrings, PROCESS_ARRAYS_AS_MULTIVALUE_STRINGS_CONFIG_STRING @@ -78,6 +82,11 @@ public class ExpressionProcessingConfig private static boolean getWithPropertyFallbackFalse(@Nullable Boolean value, String property) { - return value != null ? value : Boolean.valueOf(System.getProperty(property, "false")); + return getWithPropertyFallback(value, property, "false"); + } + + private static boolean getWithPropertyFallback(@Nullable Boolean value, String property, String fallback) + { + return value != null ? value : Boolean.valueOf(System.getProperty(property, fallback)); } } diff --git a/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java b/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java index 16ecc39c0cc..97a91cb3969 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java @@ -1382,9 +1382,6 @@ public class EvalTest extends InitializedNullHandlingTest // strings assertBestEffortOf("stringy", ExpressionType.STRING, "stringy"); - // by default, booleans are handled as strings - assertBestEffortOf(true, ExpressionType.STRING, "true"); - assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"}); assertBestEffortOf( new byte[]{1, 2, 3, 4}, @@ -1396,11 +1393,15 @@ public class EvalTest extends InitializedNullHandlingTest assertBestEffortOf(1L, ExpressionType.LONG, 1L); assertBestEffortOf(1, ExpressionType.LONG, 1L); + // by default, booleans are handled as longs + assertBestEffortOf(true, ExpressionType.LONG, 1L); + assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L}); + try { - // in strict boolean mode, they are longs - ExpressionProcessing.initializeForStrictBooleansTests(true); - assertBestEffortOf(true, ExpressionType.LONG, 1L); - assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L}); + // in non-strict boolean mode, they are strings + ExpressionProcessing.initializeForStrictBooleansTests(false); + assertBestEffortOf(true, ExpressionType.STRING, "true"); + assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"}); } finally { // reset diff --git a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index ad4dccd41a7..a032e993db0 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -26,9 +26,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.math.expr.vector.ExprEvalVector; import org.apache.druid.testing.InitializedNullHandlingTest; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; import javax.annotation.Nullable; @@ -64,18 +62,6 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest .put("boolString2", ExpressionType.STRING) .build(); - @BeforeClass - public static void setupTests() - { - ExpressionProcessing.initializeForStrictBooleansTests(true); - } - - @AfterClass - public static void teardownTests() - { - ExpressionProcessing.initializeForTests(); - } - @Test public void testUnaryOperators() { diff --git a/processing/src/test/java/org/apache/druid/query/expression/VectorExpressionsSanityTest.java b/processing/src/test/java/org/apache/druid/query/expression/VectorExpressionsSanityTest.java index 71e977909f6..ba23a54a7ca 100644 --- a/processing/src/test/java/org/apache/druid/query/expression/VectorExpressionsSanityTest.java +++ b/processing/src/test/java/org/apache/druid/query/expression/VectorExpressionsSanityTest.java @@ -26,13 +26,10 @@ import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprEval; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.VectorExprSanityTest; import org.apache.druid.testing.InitializedNullHandlingTest; import org.joda.time.DateTime; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.Test; import java.util.Map; @@ -62,18 +59,6 @@ public class VectorExpressionsSanityTest extends InitializedNullHandlingTest .put("boolString2", ExpressionType.STRING) .build(); - @BeforeClass - public static void setupTests() - { - ExpressionProcessing.initializeForStrictBooleansTests(true); - } - - @AfterClass - public static void teardownTests() - { - ExpressionProcessing.initializeForTests(); - } - static void testExpression(String expr, Expr parsed, Map types) { log.debug("[%s]", expr); diff --git a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java index b4fe9ce8698..e06b5b50335 100644 --- a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java @@ -600,7 +600,77 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest } @Test - public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypes() throws Exception + public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesNonStrictBooleans() throws Exception + { + + try { + ExpressionProcessing.initializeForStrictBooleansTests(false); + Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder() + .dataSource("test_datasource") + .intervals( + new MultipleIntervalSegmentSpec( + Collections.singletonList(Intervals.ETERNITY) + ) + ) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .limit(100) + .context(ImmutableMap.of()); + Query scanQuery = builder.build(); + final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")}; + List realtimeSegs = ImmutableList.of( + NestedDataTestUtils.createIncrementalIndex( + tempFolder, + NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2, + NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT, + NestedDataTestUtils.TIMESTAMP_SPEC, + NestedDataTestUtils.AUTO_DISCOVERY, + TransformSpec.NONE, + aggs, + Granularities.NONE, + true + ) + ); + List segs = NestedDataTestUtils.createSegments( + tempFolder, + closer, + NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2, + NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT, + NestedDataTestUtils.TIMESTAMP_SPEC, + NestedDataTestUtils.AUTO_DISCOVERY, + TransformSpec.NONE, + aggs, + Granularities.NONE, + true, + IndexSpec.DEFAULT + ); + + + final Sequence seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery); + final Sequence seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery); + + List resultsRealtime = seq.toList(); + List resultsSegments = seq2.toList(); + logResults(resultsSegments); + logResults(resultsRealtime); + Assert.assertEquals(1, resultsRealtime.size()); + Assert.assertEquals(resultsRealtime.size(), resultsSegments.size()); + Assert.assertEquals( + "[" + + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " + + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " + + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " + + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]", + resultsSegments.get(0).getEvents().toString() + ); + Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString()); + } + finally { + ExpressionProcessing.initializeForTests(); + } + } + + @Test + public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception { Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder() .dataSource("test_datasource") @@ -653,84 +723,15 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest Assert.assertEquals(resultsRealtime.size(), resultsSegments.size()); Assert.assertEquals( "[" - + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]", + + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " + + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " + + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " + + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]", resultsSegments.get(0).getEvents().toString() ); Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString()); } - @Test - public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception - { - try { - ExpressionProcessing.initializeForStrictBooleansTests(true); - Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder() - .dataSource("test_datasource") - .intervals( - new MultipleIntervalSegmentSpec( - Collections.singletonList(Intervals.ETERNITY) - ) - ) - .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) - .limit(100) - .context(ImmutableMap.of()); - Query scanQuery = builder.build(); - final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")}; - List realtimeSegs = ImmutableList.of( - NestedDataTestUtils.createIncrementalIndex( - tempFolder, - NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2, - NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT, - NestedDataTestUtils.TIMESTAMP_SPEC, - NestedDataTestUtils.AUTO_DISCOVERY, - TransformSpec.NONE, - aggs, - Granularities.NONE, - true - ) - ); - List segs = NestedDataTestUtils.createSegments( - tempFolder, - closer, - NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2, - NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT, - NestedDataTestUtils.TIMESTAMP_SPEC, - NestedDataTestUtils.AUTO_DISCOVERY, - TransformSpec.NONE, - aggs, - Granularities.NONE, - true, - IndexSpec.DEFAULT - ); - - - final Sequence seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery); - final Sequence seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery); - - List resultsRealtime = seq.toList(); - List resultsSegments = seq2.toList(); - logResults(resultsSegments); - logResults(resultsRealtime); - Assert.assertEquals(1, resultsRealtime.size()); - Assert.assertEquals(resultsRealtime.size(), resultsSegments.size()); - Assert.assertEquals( - "[" - + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], " - + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]", - resultsSegments.get(0).getEvents().toString() - ); - Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString()); - } - finally { - ExpressionProcessing.initializeForTests(); - } - } - @Test public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryTypeGauntlet() throws Exception { @@ -785,12 +786,12 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest Assert.assertEquals(resultsRealtime.size(), resultsSegments.size()); if (NullHandling.replaceWithDefault()) { Assert.assertEquals( - "[[1672531200000, null, 0, 0.0, true, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [true, false, true], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 2, 0.0, false, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, true, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [true, false, true], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, true, 1, 0.0, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, false, true], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, 0, 4.4, true, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [false], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, false, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [true, null, true], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]", + "[[1672531200000, null, 0, 0.0, 1, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 2, 0.0, 0, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [1, 1], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, 1, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [1, 0, 1], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, 1, 1, 0.0, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, 0, 1], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, 0, 4.4, 1, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [0], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, 0, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, 0, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [1, null, 1], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]", resultsSegments.get(0).getEvents().toString() ); } else { Assert.assertEquals( - "[[1672531200000, null, null, null, true, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [true, false, true], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, , 2, null, false, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, true, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [true, false, true], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, true, 1, null, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, false, true], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, null, 4.4, true, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [false], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, false, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [true, null, true], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]", + "[[1672531200000, null, null, null, 1, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, , 2, null, 0, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [1, 1], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, 1, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [1, 0, 1], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, 1, 1, null, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, 0, 1], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, null, 4.4, 1, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [0], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, 0, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [1, null, 1], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]", resultsSegments.get(0).getEvents().toString() ); } diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterStrictBooleansTest.java b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java similarity index 88% rename from processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterStrictBooleansTest.java rename to processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java index 589a7356776..3716f6c5f29 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterStrictBooleansTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java @@ -31,9 +31,9 @@ import org.junit.runners.Parameterized; import java.io.Closeable; @RunWith(Parameterized.class) -public class ExpressionFilterStrictBooleansTest extends ExpressionFilterTest +public class ExpressionFilterNonStrictBooleansTest extends ExpressionFilterTest { - public ExpressionFilterStrictBooleansTest( + public ExpressionFilterNonStrictBooleansTest( String testName, IndexBuilder indexBuilder, Function> finisher, @@ -48,6 +48,6 @@ public class ExpressionFilterStrictBooleansTest extends ExpressionFilterTest @Override public void setup() { - ExpressionProcessing.initializeForStrictBooleansTests(true); + ExpressionProcessing.initializeForStrictBooleansTests(false); } } diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java index ac0ef94c200..9c40ccb5da1 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java @@ -122,7 +122,7 @@ public class ExpressionFilterTest extends BaseFilterTest @Before public void setup() { - ExpressionProcessing.initializeForStrictBooleansTests(false); + ExpressionProcessing.initializeForStrictBooleansTests(true); } @After diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexTest.java index 9171c075294..2517ce1388f 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexTest.java @@ -400,7 +400,7 @@ public class IncrementalIndexTest extends InitializedNullHandlingTest Assert.assertEquals(ColumnType.STRING, index.getColumnCapabilities("bool_string").toColumnType()); Assert.assertEquals(ColumnType.LONG, index.getColumnCapabilities("bool_long").toColumnType()); // depends on value of 'druid.expressions.useStrictBooleans', current default is false which parses as strings - Assert.assertEquals(ColumnType.STRING, index.getColumnCapabilities("bool_auto").toColumnType()); + Assert.assertEquals(ColumnType.LONG, index.getColumnCapabilities("bool_auto").toColumnType()); Assert.assertEquals(ColumnType.STRING_ARRAY, index.getColumnCapabilities("array_string").toColumnType()); Assert.assertEquals(ColumnType.LONG_ARRAY, index.getColumnCapabilities("array_long").toColumnType()); Assert.assertEquals(ColumnType.DOUBLE_ARRAY, index.getColumnCapabilities("array_double").toColumnType()); diff --git a/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java b/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java index 4e1fad1c171..972932e2611 100644 --- a/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java +++ b/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java @@ -155,7 +155,8 @@ public class TransformerTest extends InitializedNullHandlingTest final InputRow actual = transformer.transform(row); Assert.assertNotNull(actual); Assert.assertEquals(ImmutableList.of("dim"), actual.getDimensions()); - Assert.assertEquals(5L, actual.getRaw("dim")); + // booleans are longs by default, so strlen of false (0L) is 1 + Assert.assertEquals(1L, actual.getRaw("dim")); Assert.assertEquals(row.getTimestamp(), actual.getTimestamp()); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index bb3dda46f73..406f94b4643 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -5661,7 +5661,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "", 0L, 0.0D, - "true", + 1L, "51", -0.13D, "1", @@ -5676,7 +5676,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest null, "[null]", null, - "[\"true\",\"false\",\"true\"]", + "[1,0,1]", null, "[{\"x\":1},{\"x\":2}]", "", @@ -5699,7 +5699,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "", 2L, 0.0D, - "false", + 0L, "b", 1.1D, "\"b\"", @@ -5714,7 +5714,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[3.3,4.4,5.5]", "[999.0,null,5.5]", "[null,null,2.2]", - "[\"true\",\"true\"]", + "[1,1]", "[null,[null],[]]", "[{\"x\":3},{\"x\":4}]", "", @@ -5737,7 +5737,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "a", 1L, 1.0D, - "true", + 1L, "1", 1.0D, "1", @@ -5752,7 +5752,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[1.1,2.2,3.3]", "[1.1,2.2,null]", "[\"a\",\"1\",\"2.2\"]", - "[\"true\",\"false\",\"true\"]", + "[1,0,1]", "[[1,2,null],[3,4]]", "[{\"x\":1},{\"x\":2}]", "", @@ -5775,7 +5775,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "b", 4L, 3.3D, - "true", + 1L, "1", 0.0D, "{}", @@ -5790,7 +5790,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[2.2,3.3,4.0]", null, "[\"a\",\"b\",\"c\"]", - "[null,\"false\",\"true\"]", + "[null,0,1]", "[[1,2],[3,4],[5,6,7]]", "[{\"x\":null},{\"x\":2}]", "", @@ -5813,7 +5813,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "c", 0L, 4.4D, - "true", + 1L, "hello", -1000.0D, "{}", @@ -5828,7 +5828,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[1.1,2.2,3.3]", null, null, - "[\"false\"]", + "[0]", null, "[{\"x\":1000},{\"y\":2000}]", "", @@ -5851,7 +5851,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "d", 5L, 5.9D, - "false", + 0L, "", 3.33D, "\"a\"", @@ -5889,7 +5889,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "null", 3L, 2.0D, - "", + 0L, "3.0", 1.0D, "3.3", @@ -5904,7 +5904,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[1.1,3.3]", "[null,2.2,null]", "[1,null,1]", - "[\"true\",null,\"true\"]", + "[1,null,1]", "[[1],null,[1,2,3]]", "[null,{\"x\":2}]", "", @@ -5929,7 +5929,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest null, null, null, - "true", + 1L, "51", -0.13D, "1", @@ -5944,7 +5944,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest null, "[null]", null, - "[\"true\",\"false\",\"true\"]", + "[1,0,1]", null, "[{\"x\":1},{\"x\":2}]", null, @@ -5967,7 +5967,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "", 2L, null, - "false", + 0L, "b", 1.1D, "\"b\"", @@ -5982,7 +5982,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[3.3,4.4,5.5]", "[999.0,null,5.5]", "[null,null,2.2]", - "[\"true\",\"true\"]", + "[1,1]", "[null,[null],[]]", "[{\"x\":3},{\"x\":4}]", null, @@ -6005,7 +6005,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "a", 1L, 1.0D, - "true", + 1L, "1", 1.0D, "1", @@ -6020,7 +6020,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[1.1,2.2,3.3]", "[1.1,2.2,null]", "[\"a\",\"1\",\"2.2\"]", - "[\"true\",\"false\",\"true\"]", + "[1,0,1]", "[[1,2,null],[3,4]]", "[{\"x\":1},{\"x\":2}]", null, @@ -6043,7 +6043,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "b", 4L, 3.3D, - "true", + 1L, "1", null, "{}", @@ -6058,7 +6058,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[2.2,3.3,4.0]", null, "[\"a\",\"b\",\"c\"]", - "[null,\"false\",\"true\"]", + "[null,0,1]", "[[1,2],[3,4],[5,6,7]]", "[{\"x\":null},{\"x\":2}]", null, @@ -6081,7 +6081,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "c", null, 4.4D, - "true", + 1L, "hello", -1000.0D, "{}", @@ -6096,7 +6096,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[1.1,2.2,3.3]", null, null, - "[\"false\"]", + "[0]", null, "[{\"x\":1000},{\"y\":2000}]", null, @@ -6119,7 +6119,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "d", 5L, 5.9D, - "false", + 0L, null, 3.33D, "\"a\"", @@ -6172,7 +6172,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest "[1.1,3.3]", "[null,2.2,null]", "[1,null,1]", - "[\"true\",null,\"true\"]", + "[1,null,1]", "[[1],null,[1,2,3]]", "[null,{\"x\":2}]", null, @@ -6196,7 +6196,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest .add("str", ColumnType.STRING) .add("long", ColumnType.LONG) .add("double", ColumnType.DOUBLE) - .add("bool", ColumnType.STRING) + .add("bool", ColumnType.LONG) .add("variant", ColumnType.STRING) .add("variantNumeric", ColumnType.DOUBLE) .add("variantEmptyObj", ColumnType.NESTED_DATA) @@ -6211,7 +6211,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest .add("arrayDouble", ColumnType.DOUBLE_ARRAY) .add("arrayDoubleNulls", ColumnType.DOUBLE_ARRAY) .add("arrayVariant", ColumnType.STRING_ARRAY) - .add("arrayBool", ColumnType.STRING_ARRAY) + .add("arrayBool", ColumnType.LONG_ARRAY) .add("arrayNestedLong", ColumnType.NESTED_DATA) .add("arrayObject", ColumnType.NESTED_DATA) .add("null", ColumnType.STRING) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java index 157a9e27175..2ac0574c278 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java @@ -29,7 +29,6 @@ import org.apache.druid.java.util.common.guava.Yielder; import org.apache.druid.java.util.common.guava.Yielders; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.QueryableIndex; @@ -113,7 +112,6 @@ public class SqlVectorizedExpressionSanityTest extends InitializedNullHandlingTe @BeforeClass public static void setupClass() { - ExpressionProcessing.initializeForStrictBooleansTests(true); CLOSER = Closer.create(); final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench"); @@ -162,7 +160,6 @@ public class SqlVectorizedExpressionSanityTest extends InitializedNullHandlingTe public static void teardownClass() throws IOException { CLOSER.close(); - ExpressionProcessing.initializeForTests(); } @Parameterized.Parameters(name = "query = {0}")