mirror of https://github.com/apache/druid.git
set druid.expressions.useStrictBooleans to true by default (#14734)
This commit is contained in:
parent
d87056e708
commit
194a9c9abc
|
@ -84,7 +84,7 @@ public class SqlExpressionBenchmark
|
|||
|
||||
static {
|
||||
NullHandling.initializeForTests();
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
ExpressionProcessing.initializeForTests();
|
||||
}
|
||||
|
||||
private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig()
|
||||
|
|
|
@ -90,7 +90,7 @@ public class SqlNestedDataBenchmark
|
|||
|
||||
static {
|
||||
NullHandling.initializeForTests();
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
ExpressionProcessing.initializeForTests();
|
||||
}
|
||||
|
||||
private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig()
|
||||
|
|
|
@ -2288,8 +2288,8 @@ Supported query contexts:
|
|||
|
||||
|Key|Description|Default|
|
||||
|---|-----------|-------|
|
||||
|`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean operators and functions, if set to `true` all boolean values will be either a `1` or `0`. See [expression documentation](../querying/math-expr.md#logical-operator-modes)|false|
|
||||
|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can create nested arrays.|false|
|
||||
|`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean operators and functions, if set to `true` all boolean values are either `1` or `0`. See [expression documentation](../querying/math-expr.md#logical-operator-modes) for more information.|true|
|
||||
|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can create nested arrays.|true|
|
||||
### Router
|
||||
|
||||
#### Router Process Configs
|
||||
|
|
|
@ -57,4 +57,3 @@ Note that this document does not track the status of contrib extensions, all of
|
|||
|
||||
- [Configuration reference](../configuration/index.md)
|
||||
- `CLOSED_SEGMENTS_SINKS` mode
|
||||
- Expression processing configuration `druid.expressions.allowNestedArrays`
|
||||
|
|
|
@ -258,8 +258,8 @@ You can have Druid infer the schema and types for your data partially or fully b
|
|||
When performing type-aware schema discovery, Druid can discover all of the columns of your input data (that aren't in
|
||||
the exclusion list). Druid automatically chooses the most appropriate native Druid type among `STRING`, `LONG`,
|
||||
`DOUBLE`, `ARRAY<STRING>`, `ARRAY<LONG>`, `ARRAY<DOUBLE>`, or `COMPLEX<json>` for nested data. For input formats with
|
||||
native boolean types, Druid ingests these values as strings if `druid.expressions.useStrictBooleans` is set to `false`
|
||||
(the default), or longs if set to `true` (for more SQL compatible behavior). Array typed columns can be queried using
|
||||
native boolean types, Druid ingests these values as longs if `druid.expressions.useStrictBooleans` is set to `true`
|
||||
(the default) or strings if set to `false`. Array typed columns can be queried using
|
||||
the [array functions](../querying/sql-array-functions.md) or [UNNEST](../querying/sql-functions.md#unnest). Nested
|
||||
columns can be queried with the [JSON functions](../querying/sql-json-functions.md).
|
||||
|
||||
|
|
|
@ -299,7 +299,7 @@ Supported features:
|
|||
* constants and identifiers are supported for any column type
|
||||
* `cast` is supported for numeric and string types
|
||||
* math operators: `+`,`-`,`*`,`/`,`%`,`^` are supported for numeric types
|
||||
* logical operators: `!`, `&&`, `||`, are supported for string and numeric types (if `druid.expressions.useStrictBooleans=true`)
|
||||
* logical operators: `!`, `&&`, `||`, are supported for string and numeric types
|
||||
* comparison operators: `=`, `!=`, `>`, `>=`, `<`, `<=` are supported for string and numeric types
|
||||
* math functions: `abs`, `acos`, `asin`, `atan`, `cbrt`, `ceil`, `cos`, `cosh`, `cot`, `exp`, `expm1`, `floor`, `getExponent`, `log`, `log10`, `log1p`, `nextUp`, `rint`, `signum`, `sin`, `sinh`, `sqrt`, `tan`, `tanh`, `toDegrees`, `toRadians`, `ulp`, `atan2`, `copySign`, `div`, `hypot`, `max`, `min`, `nextAfter`, `pow`, `remainder`, `scalb` are supported for numeric types
|
||||
* time functions: `timestamp_floor` (with constant granularity argument) is supported for numeric types
|
||||
|
@ -309,9 +309,7 @@ Supported features:
|
|||
* other: `parse_long` is supported for numeric and string types
|
||||
|
||||
## Logical operator modes
|
||||
Prior to the 0.23 release of Apache Druid, boolean function expressions have inconsistent handling of true and false values, and the logical 'and' and 'or' operators behave in a manner that is incompatible with SQL, even if SQL compatible null handling mode (`druid.generic.useDefaultValueForNull=false`) is enabled. Logical operators also pass through their input values similar to many scripting languages, and treat `null` as false, which can result in some rather strange behavior. Other boolean operations, such as comparisons and equality, retain their input types (e.g. `DOUBLE` comparison would produce `1.0` for true and `0.0` for false), while many other boolean functions strictly produce `LONG` typed values of `1` for true and `0` for false.
|
||||
|
||||
After 0.23, while the inconsistent legacy behavior is still the default, it can be optionally be changed by setting `druid.expressions.useStrictBooleans=true`, so that these operations will allow correctly treating `null` values as "unknown" for SQL compatible behavior, and _all boolean output functions_ will output 'homogeneous' `LONG` typed boolean values of `1` for `true` and `0` for `false`. Additionally,
|
||||
In Druid 28.0 and later, `druid.expressions.useStrictBooleans=true` is set by default. Logical operations treat `null` values as "unknown" for SQL compatible behavior. _All boolean output functions_ will output 'homogeneous' `LONG` typed boolean values of `1` for `true` and `0` for `false`.
|
||||
|
||||
For the "or" operator:
|
||||
* `true || null`, `null || true`, -> `1`
|
||||
|
@ -322,15 +320,8 @@ For the "and" operator:
|
|||
* `false && null`, `null && false` -> `0`
|
||||
|
||||
Druid currently still retains implicit conversion of `LONG`, `DOUBLE`, and `STRING` types into boolean values in both modes:
|
||||
* `LONG` or `DOUBLE` - any value greater than 0 is considered `true`, else `false`
|
||||
* `STRING` - the value `'true'` (case insensitive) is considered `true`, everything else is `false`.
|
||||
|
||||
Legacy behavior:
|
||||
* `100 && 11` -> `11`
|
||||
* `0.7 || 0.3` -> `0.3`
|
||||
* `100 && 0` -> `0`
|
||||
* `'troo' && 'true'` -> `'troo'`
|
||||
* `'troo' || 'true'` -> `'true'`
|
||||
* `LONG` or `DOUBLE`: any value greater than 0 is considered `true`, else `false`.
|
||||
* `STRING`: the value `'true'` (case insensitive) is considered `true`, everything else is `false`.
|
||||
|
||||
SQL compatible behavior:
|
||||
* `100 && 11` -> `1`
|
||||
|
@ -339,4 +330,11 @@ SQL compatible behavior:
|
|||
* `'troo' && 'true'` -> `0`
|
||||
* `'troo' || 'true'` -> `1`
|
||||
|
||||
Prior to Druid 28.0.0, `druid.expressions.useStrictBooleans=false` was the default. In this mode, boolean function expressions have inconsistent handling of true and false values. The logical 'and' and 'or' operators behave in a manner that is incompatible with SQL, even if SQL compatible null handling mode (`druid.generic.useDefaultValueForNull=false`) is enabled. Logical operators also pass through their input values, similar to many scripting languages, and treat `null` as false, which results in some rather strange behavior. Other boolean operations, such as comparisons and equality, retain their input types (e.g. `DOUBLE` comparison produces `1.0` for true and `0.0` for false), while many other boolean functions strictly produce `LONG` typed values of `1` for true and `0` for false. This legacy mode can still be enabled by setting `druid.expressions.useStrictBooleans=false`.
|
||||
|
||||
Legacy behavior:
|
||||
* `100 && 11` -> `11`
|
||||
* `0.7 || 0.3` -> `0.3`
|
||||
* `100 && 0` -> `0`
|
||||
* `'troo' && 'true'` -> `'troo'`
|
||||
* `'troo' || 'true'` -> `'true'`
|
||||
|
|
|
@ -153,13 +153,13 @@ values are treated as zeroes. This was the default prior to Druid 28.0.0.
|
|||
## Boolean logic
|
||||
|
||||
The [`druid.expressions.useStrictBooleans`](../configuration/index.md#expression-processing-configurations)
|
||||
runtime property controls Druid's boolean logic mode. For the most SQL compliant behavior, set this to `true`.
|
||||
runtime property controls Druid's boolean logic mode. For the most SQL compliant behavior, set this to `true` (the default).
|
||||
|
||||
When `druid.expressions.useStrictBooleans = false` (the default mode), Druid uses three-valued logic for
|
||||
When `druid.expressions.useStrictBooleans = true`, Druid uses three-valued logic for
|
||||
[expressions](math-expr.md) evaluation, such as `expression` virtual columns or `expression` filters.
|
||||
However, even in this mode, Druid uses two-valued logic for filter types other than `expression`.
|
||||
|
||||
When `druid.expressions.useStrictBooleans = true` (legacy mode), Druid uses two-valued logic.
|
||||
When `druid.expressions.useStrictBooleans = false` (legacy mode), Druid uses two-valued logic.
|
||||
|
||||
## Nested columns
|
||||
|
||||
|
|
|
@ -58,71 +58,11 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest
|
|||
private InputSourceSampler inputSourceSampler = new InputSourceSampler(OBJECT_MAPPER);
|
||||
|
||||
@Test
|
||||
public void testDiscoveredTypes()
|
||||
public void testDiscoveredTypesNonStrictBooleans()
|
||||
{
|
||||
final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
|
||||
final SamplerResponse response = inputSourceSampler.sample(
|
||||
inputSource,
|
||||
new JsonInputFormat(null, null, null, null, null),
|
||||
new DataSchema(
|
||||
"test",
|
||||
new TimestampSpec("t", null, null),
|
||||
DimensionsSpec.builder().useSchemaDiscovery(true).build(),
|
||||
null,
|
||||
null,
|
||||
null
|
||||
),
|
||||
null
|
||||
);
|
||||
|
||||
Assert.assertEquals(6, response.getNumRowsRead());
|
||||
Assert.assertEquals(5, response.getNumRowsIndexed());
|
||||
Assert.assertEquals(6, response.getData().size());
|
||||
Assert.assertEquals(
|
||||
ImmutableList.of(
|
||||
new StringDimensionSchema("string"),
|
||||
new LongDimensionSchema("long"),
|
||||
new DoubleDimensionSchema("double"),
|
||||
new StringDimensionSchema("bool"),
|
||||
new StringDimensionSchema("variant"),
|
||||
new AutoTypeColumnSchema("array"),
|
||||
new AutoTypeColumnSchema("nested")
|
||||
),
|
||||
response.getLogicalDimensions()
|
||||
);
|
||||
|
||||
Assert.assertEquals(
|
||||
ImmutableList.of(
|
||||
new AutoTypeColumnSchema("string"),
|
||||
new AutoTypeColumnSchema("long"),
|
||||
new AutoTypeColumnSchema("double"),
|
||||
new AutoTypeColumnSchema("bool"),
|
||||
new AutoTypeColumnSchema("variant"),
|
||||
new AutoTypeColumnSchema("array"),
|
||||
new AutoTypeColumnSchema("nested")
|
||||
),
|
||||
response.getPhysicalDimensions()
|
||||
);
|
||||
Assert.assertEquals(
|
||||
RowSignature.builder()
|
||||
.addTimeColumn()
|
||||
.add("string", ColumnType.STRING)
|
||||
.add("long", ColumnType.LONG)
|
||||
.add("double", ColumnType.DOUBLE)
|
||||
.add("bool", ColumnType.STRING)
|
||||
.add("variant", ColumnType.STRING)
|
||||
.add("array", ColumnType.LONG_ARRAY)
|
||||
.add("nested", ColumnType.NESTED_DATA)
|
||||
.build(),
|
||||
response.getLogicalSegmentSchema()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDiscoveredTypesStrictBooleans()
|
||||
{
|
||||
try {
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(false);
|
||||
final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
|
||||
final SamplerResponse response = inputSourceSampler.sample(
|
||||
inputSource,
|
||||
|
@ -146,7 +86,7 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest
|
|||
new StringDimensionSchema("string"),
|
||||
new LongDimensionSchema("long"),
|
||||
new DoubleDimensionSchema("double"),
|
||||
new LongDimensionSchema("bool"),
|
||||
new StringDimensionSchema("bool"),
|
||||
new StringDimensionSchema("variant"),
|
||||
new AutoTypeColumnSchema("array"),
|
||||
new AutoTypeColumnSchema("nested")
|
||||
|
@ -172,7 +112,7 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest
|
|||
.add("string", ColumnType.STRING)
|
||||
.add("long", ColumnType.LONG)
|
||||
.add("double", ColumnType.DOUBLE)
|
||||
.add("bool", ColumnType.LONG)
|
||||
.add("bool", ColumnType.STRING)
|
||||
.add("variant", ColumnType.STRING)
|
||||
.add("array", ColumnType.LONG_ARRAY)
|
||||
.add("nested", ColumnType.NESTED_DATA)
|
||||
|
@ -185,6 +125,67 @@ public class InputSourceSamplerDiscoveryTest extends InitializedNullHandlingTest
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDiscoveredTypesStrictBooleans()
|
||||
{
|
||||
final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
|
||||
final SamplerResponse response = inputSourceSampler.sample(
|
||||
inputSource,
|
||||
new JsonInputFormat(null, null, null, null, null),
|
||||
new DataSchema(
|
||||
"test",
|
||||
new TimestampSpec("t", null, null),
|
||||
DimensionsSpec.builder().useSchemaDiscovery(true).build(),
|
||||
null,
|
||||
null,
|
||||
null
|
||||
),
|
||||
null
|
||||
);
|
||||
|
||||
Assert.assertEquals(6, response.getNumRowsRead());
|
||||
Assert.assertEquals(5, response.getNumRowsIndexed());
|
||||
Assert.assertEquals(6, response.getData().size());
|
||||
Assert.assertEquals(
|
||||
ImmutableList.of(
|
||||
new StringDimensionSchema("string"),
|
||||
new LongDimensionSchema("long"),
|
||||
new DoubleDimensionSchema("double"),
|
||||
new LongDimensionSchema("bool"),
|
||||
new StringDimensionSchema("variant"),
|
||||
new AutoTypeColumnSchema("array"),
|
||||
new AutoTypeColumnSchema("nested")
|
||||
),
|
||||
response.getLogicalDimensions()
|
||||
);
|
||||
|
||||
Assert.assertEquals(
|
||||
ImmutableList.of(
|
||||
new AutoTypeColumnSchema("string"),
|
||||
new AutoTypeColumnSchema("long"),
|
||||
new AutoTypeColumnSchema("double"),
|
||||
new AutoTypeColumnSchema("bool"),
|
||||
new AutoTypeColumnSchema("variant"),
|
||||
new AutoTypeColumnSchema("array"),
|
||||
new AutoTypeColumnSchema("nested")
|
||||
),
|
||||
response.getPhysicalDimensions()
|
||||
);
|
||||
Assert.assertEquals(
|
||||
RowSignature.builder()
|
||||
.addTimeColumn()
|
||||
.add("string", ColumnType.STRING)
|
||||
.add("long", ColumnType.LONG)
|
||||
.add("double", ColumnType.DOUBLE)
|
||||
.add("bool", ColumnType.LONG)
|
||||
.add("variant", ColumnType.STRING)
|
||||
.add("array", ColumnType.LONG_ARRAY)
|
||||
.add("nested", ColumnType.NESTED_DATA)
|
||||
.build(),
|
||||
response.getLogicalSegmentSchema()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTypesClassicDiscovery()
|
||||
{
|
||||
|
|
|
@ -561,9 +561,6 @@ public abstract class ExprEval<T>
|
|||
return ofDouble((Number) value);
|
||||
}
|
||||
if (value instanceof Boolean) {
|
||||
if (ExpressionProcessing.useStrictBooleans()) {
|
||||
return ofLongBoolean((Boolean) value);
|
||||
}
|
||||
return ofDouble(Evals.asDouble((Boolean) value));
|
||||
}
|
||||
if (value instanceof String) {
|
||||
|
|
|
@ -50,7 +50,11 @@ public class ExpressionProcessingConfig
|
|||
@JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays
|
||||
)
|
||||
{
|
||||
this.useStrictBooleans = getWithPropertyFallbackFalse(useStrictBooleans, NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING);
|
||||
this.useStrictBooleans = getWithPropertyFallback(
|
||||
useStrictBooleans,
|
||||
NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING,
|
||||
"true"
|
||||
);
|
||||
this.processArraysAsMultiValueStrings = getWithPropertyFallbackFalse(
|
||||
processArraysAsMultiValueStrings,
|
||||
PROCESS_ARRAYS_AS_MULTIVALUE_STRINGS_CONFIG_STRING
|
||||
|
@ -78,6 +82,11 @@ public class ExpressionProcessingConfig
|
|||
|
||||
private static boolean getWithPropertyFallbackFalse(@Nullable Boolean value, String property)
|
||||
{
|
||||
return value != null ? value : Boolean.valueOf(System.getProperty(property, "false"));
|
||||
return getWithPropertyFallback(value, property, "false");
|
||||
}
|
||||
|
||||
private static boolean getWithPropertyFallback(@Nullable Boolean value, String property, String fallback)
|
||||
{
|
||||
return value != null ? value : Boolean.valueOf(System.getProperty(property, fallback));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1382,9 +1382,6 @@ public class EvalTest extends InitializedNullHandlingTest
|
|||
// strings
|
||||
assertBestEffortOf("stringy", ExpressionType.STRING, "stringy");
|
||||
|
||||
// by default, booleans are handled as strings
|
||||
assertBestEffortOf(true, ExpressionType.STRING, "true");
|
||||
assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"});
|
||||
|
||||
assertBestEffortOf(
|
||||
new byte[]{1, 2, 3, 4},
|
||||
|
@ -1396,11 +1393,15 @@ public class EvalTest extends InitializedNullHandlingTest
|
|||
assertBestEffortOf(1L, ExpressionType.LONG, 1L);
|
||||
assertBestEffortOf(1, ExpressionType.LONG, 1L);
|
||||
|
||||
// by default, booleans are handled as longs
|
||||
assertBestEffortOf(true, ExpressionType.LONG, 1L);
|
||||
assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L});
|
||||
|
||||
try {
|
||||
// in strict boolean mode, they are longs
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
assertBestEffortOf(true, ExpressionType.LONG, 1L);
|
||||
assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L});
|
||||
// in non-strict boolean mode, they are strings
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(false);
|
||||
assertBestEffortOf(true, ExpressionType.STRING, "true");
|
||||
assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"});
|
||||
}
|
||||
finally {
|
||||
// reset
|
||||
|
|
|
@ -26,9 +26,7 @@ import org.apache.druid.java.util.common.StringUtils;
|
|||
import org.apache.druid.java.util.common.logger.Logger;
|
||||
import org.apache.druid.math.expr.vector.ExprEvalVector;
|
||||
import org.apache.druid.testing.InitializedNullHandlingTest;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
@ -64,18 +62,6 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest
|
|||
.put("boolString2", ExpressionType.STRING)
|
||||
.build();
|
||||
|
||||
@BeforeClass
|
||||
public static void setupTests()
|
||||
{
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void teardownTests()
|
||||
{
|
||||
ExpressionProcessing.initializeForTests();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUnaryOperators()
|
||||
{
|
||||
|
|
|
@ -26,13 +26,10 @@ import org.apache.druid.java.util.common.NonnullPair;
|
|||
import org.apache.druid.java.util.common.logger.Logger;
|
||||
import org.apache.druid.math.expr.Expr;
|
||||
import org.apache.druid.math.expr.ExprEval;
|
||||
import org.apache.druid.math.expr.ExpressionProcessing;
|
||||
import org.apache.druid.math.expr.ExpressionType;
|
||||
import org.apache.druid.math.expr.VectorExprSanityTest;
|
||||
import org.apache.druid.testing.InitializedNullHandlingTest;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.Map;
|
||||
|
@ -62,18 +59,6 @@ public class VectorExpressionsSanityTest extends InitializedNullHandlingTest
|
|||
.put("boolString2", ExpressionType.STRING)
|
||||
.build();
|
||||
|
||||
@BeforeClass
|
||||
public static void setupTests()
|
||||
{
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void teardownTests()
|
||||
{
|
||||
ExpressionProcessing.initializeForTests();
|
||||
}
|
||||
|
||||
static void testExpression(String expr, Expr parsed, Map<String, ExpressionType> types)
|
||||
{
|
||||
log.debug("[%s]", expr);
|
||||
|
|
|
@ -600,7 +600,77 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypes() throws Exception
|
||||
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesNonStrictBooleans() throws Exception
|
||||
{
|
||||
|
||||
try {
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(false);
|
||||
Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
|
||||
.dataSource("test_datasource")
|
||||
.intervals(
|
||||
new MultipleIntervalSegmentSpec(
|
||||
Collections.singletonList(Intervals.ETERNITY)
|
||||
)
|
||||
)
|
||||
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
|
||||
.limit(100)
|
||||
.context(ImmutableMap.of());
|
||||
Query<ScanResultValue> scanQuery = builder.build();
|
||||
final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")};
|
||||
List<Segment> realtimeSegs = ImmutableList.of(
|
||||
NestedDataTestUtils.createIncrementalIndex(
|
||||
tempFolder,
|
||||
NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
|
||||
NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT,
|
||||
NestedDataTestUtils.TIMESTAMP_SPEC,
|
||||
NestedDataTestUtils.AUTO_DISCOVERY,
|
||||
TransformSpec.NONE,
|
||||
aggs,
|
||||
Granularities.NONE,
|
||||
true
|
||||
)
|
||||
);
|
||||
List<Segment> segs = NestedDataTestUtils.createSegments(
|
||||
tempFolder,
|
||||
closer,
|
||||
NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
|
||||
NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT,
|
||||
NestedDataTestUtils.TIMESTAMP_SPEC,
|
||||
NestedDataTestUtils.AUTO_DISCOVERY,
|
||||
TransformSpec.NONE,
|
||||
aggs,
|
||||
Granularities.NONE,
|
||||
true,
|
||||
IndexSpec.DEFAULT
|
||||
);
|
||||
|
||||
|
||||
final Sequence<ScanResultValue> seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery);
|
||||
final Sequence<ScanResultValue> seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery);
|
||||
|
||||
List<ScanResultValue> resultsRealtime = seq.toList();
|
||||
List<ScanResultValue> resultsSegments = seq2.toList();
|
||||
logResults(resultsSegments);
|
||||
logResults(resultsRealtime);
|
||||
Assert.assertEquals(1, resultsRealtime.size());
|
||||
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
|
||||
Assert.assertEquals(
|
||||
"["
|
||||
+ "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
|
||||
resultsSegments.get(0).getEvents().toString()
|
||||
);
|
||||
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
|
||||
}
|
||||
finally {
|
||||
ExpressionProcessing.initializeForTests();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception
|
||||
{
|
||||
Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
|
||||
.dataSource("test_datasource")
|
||||
|
@ -653,84 +723,15 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest
|
|||
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
|
||||
Assert.assertEquals(
|
||||
"["
|
||||
+ "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
|
||||
+ "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
|
||||
resultsSegments.get(0).getEvents().toString()
|
||||
);
|
||||
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception
|
||||
{
|
||||
try {
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
|
||||
.dataSource("test_datasource")
|
||||
.intervals(
|
||||
new MultipleIntervalSegmentSpec(
|
||||
Collections.singletonList(Intervals.ETERNITY)
|
||||
)
|
||||
)
|
||||
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
|
||||
.limit(100)
|
||||
.context(ImmutableMap.of());
|
||||
Query<ScanResultValue> scanQuery = builder.build();
|
||||
final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")};
|
||||
List<Segment> realtimeSegs = ImmutableList.of(
|
||||
NestedDataTestUtils.createIncrementalIndex(
|
||||
tempFolder,
|
||||
NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
|
||||
NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT,
|
||||
NestedDataTestUtils.TIMESTAMP_SPEC,
|
||||
NestedDataTestUtils.AUTO_DISCOVERY,
|
||||
TransformSpec.NONE,
|
||||
aggs,
|
||||
Granularities.NONE,
|
||||
true
|
||||
)
|
||||
);
|
||||
List<Segment> segs = NestedDataTestUtils.createSegments(
|
||||
tempFolder,
|
||||
closer,
|
||||
NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
|
||||
NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT,
|
||||
NestedDataTestUtils.TIMESTAMP_SPEC,
|
||||
NestedDataTestUtils.AUTO_DISCOVERY,
|
||||
TransformSpec.NONE,
|
||||
aggs,
|
||||
Granularities.NONE,
|
||||
true,
|
||||
IndexSpec.DEFAULT
|
||||
);
|
||||
|
||||
|
||||
final Sequence<ScanResultValue> seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery);
|
||||
final Sequence<ScanResultValue> seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery);
|
||||
|
||||
List<ScanResultValue> resultsRealtime = seq.toList();
|
||||
List<ScanResultValue> resultsSegments = seq2.toList();
|
||||
logResults(resultsSegments);
|
||||
logResults(resultsRealtime);
|
||||
Assert.assertEquals(1, resultsRealtime.size());
|
||||
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
|
||||
Assert.assertEquals(
|
||||
"["
|
||||
+ "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
|
||||
+ "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
|
||||
resultsSegments.get(0).getEvents().toString()
|
||||
);
|
||||
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
|
||||
}
|
||||
finally {
|
||||
ExpressionProcessing.initializeForTests();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryTypeGauntlet() throws Exception
|
||||
{
|
||||
|
@ -785,12 +786,12 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest
|
|||
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
|
||||
if (NullHandling.replaceWithDefault()) {
|
||||
Assert.assertEquals(
|
||||
"[[1672531200000, null, 0, 0.0, true, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [true, false, true], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 2, 0.0, false, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, true, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [true, false, true], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, true, 1, 0.0, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, false, true], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, 0, 4.4, true, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [false], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, false, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [true, null, true], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
|
||||
"[[1672531200000, null, 0, 0.0, 1, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 2, 0.0, 0, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [1, 1], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, 1, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [1, 0, 1], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, 1, 1, 0.0, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, 0, 1], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, 0, 4.4, 1, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [0], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, 0, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, 0, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [1, null, 1], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
|
||||
resultsSegments.get(0).getEvents().toString()
|
||||
);
|
||||
} else {
|
||||
Assert.assertEquals(
|
||||
"[[1672531200000, null, null, null, true, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [true, false, true], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, , 2, null, false, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, true, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [true, false, true], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, true, 1, null, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, false, true], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, null, 4.4, true, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [false], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, false, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [true, null, true], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
|
||||
"[[1672531200000, null, null, null, 1, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, , 2, null, 0, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [1, 1], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, 1, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [1, 0, 1], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, 1, 1, null, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, 0, 1], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, null, 4.4, 1, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [0], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, 0, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [1, null, 1], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
|
||||
resultsSegments.get(0).getEvents().toString()
|
||||
);
|
||||
}
|
||||
|
|
|
@ -31,9 +31,9 @@ import org.junit.runners.Parameterized;
|
|||
import java.io.Closeable;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class ExpressionFilterStrictBooleansTest extends ExpressionFilterTest
|
||||
public class ExpressionFilterNonStrictBooleansTest extends ExpressionFilterTest
|
||||
{
|
||||
public ExpressionFilterStrictBooleansTest(
|
||||
public ExpressionFilterNonStrictBooleansTest(
|
||||
String testName,
|
||||
IndexBuilder indexBuilder,
|
||||
Function<IndexBuilder, Pair<StorageAdapter, Closeable>> finisher,
|
||||
|
@ -48,6 +48,6 @@ public class ExpressionFilterStrictBooleansTest extends ExpressionFilterTest
|
|||
@Override
|
||||
public void setup()
|
||||
{
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(false);
|
||||
}
|
||||
}
|
|
@ -122,7 +122,7 @@ public class ExpressionFilterTest extends BaseFilterTest
|
|||
@Before
|
||||
public void setup()
|
||||
{
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(false);
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
}
|
||||
|
||||
@After
|
||||
|
|
|
@ -400,7 +400,7 @@ public class IncrementalIndexTest extends InitializedNullHandlingTest
|
|||
Assert.assertEquals(ColumnType.STRING, index.getColumnCapabilities("bool_string").toColumnType());
|
||||
Assert.assertEquals(ColumnType.LONG, index.getColumnCapabilities("bool_long").toColumnType());
|
||||
// depends on value of 'druid.expressions.useStrictBooleans', current default is false which parses as strings
|
||||
Assert.assertEquals(ColumnType.STRING, index.getColumnCapabilities("bool_auto").toColumnType());
|
||||
Assert.assertEquals(ColumnType.LONG, index.getColumnCapabilities("bool_auto").toColumnType());
|
||||
Assert.assertEquals(ColumnType.STRING_ARRAY, index.getColumnCapabilities("array_string").toColumnType());
|
||||
Assert.assertEquals(ColumnType.LONG_ARRAY, index.getColumnCapabilities("array_long").toColumnType());
|
||||
Assert.assertEquals(ColumnType.DOUBLE_ARRAY, index.getColumnCapabilities("array_double").toColumnType());
|
||||
|
|
|
@ -155,7 +155,8 @@ public class TransformerTest extends InitializedNullHandlingTest
|
|||
final InputRow actual = transformer.transform(row);
|
||||
Assert.assertNotNull(actual);
|
||||
Assert.assertEquals(ImmutableList.of("dim"), actual.getDimensions());
|
||||
Assert.assertEquals(5L, actual.getRaw("dim"));
|
||||
// booleans are longs by default, so strlen of false (0L) is 1
|
||||
Assert.assertEquals(1L, actual.getRaw("dim"));
|
||||
Assert.assertEquals(row.getTimestamp(), actual.getTimestamp());
|
||||
}
|
||||
|
||||
|
|
|
@ -5661,7 +5661,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"",
|
||||
0L,
|
||||
0.0D,
|
||||
"true",
|
||||
1L,
|
||||
"51",
|
||||
-0.13D,
|
||||
"1",
|
||||
|
@ -5676,7 +5676,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
null,
|
||||
"[null]",
|
||||
null,
|
||||
"[\"true\",\"false\",\"true\"]",
|
||||
"[1,0,1]",
|
||||
null,
|
||||
"[{\"x\":1},{\"x\":2}]",
|
||||
"",
|
||||
|
@ -5699,7 +5699,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"",
|
||||
2L,
|
||||
0.0D,
|
||||
"false",
|
||||
0L,
|
||||
"b",
|
||||
1.1D,
|
||||
"\"b\"",
|
||||
|
@ -5714,7 +5714,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[3.3,4.4,5.5]",
|
||||
"[999.0,null,5.5]",
|
||||
"[null,null,2.2]",
|
||||
"[\"true\",\"true\"]",
|
||||
"[1,1]",
|
||||
"[null,[null],[]]",
|
||||
"[{\"x\":3},{\"x\":4}]",
|
||||
"",
|
||||
|
@ -5737,7 +5737,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"a",
|
||||
1L,
|
||||
1.0D,
|
||||
"true",
|
||||
1L,
|
||||
"1",
|
||||
1.0D,
|
||||
"1",
|
||||
|
@ -5752,7 +5752,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[1.1,2.2,3.3]",
|
||||
"[1.1,2.2,null]",
|
||||
"[\"a\",\"1\",\"2.2\"]",
|
||||
"[\"true\",\"false\",\"true\"]",
|
||||
"[1,0,1]",
|
||||
"[[1,2,null],[3,4]]",
|
||||
"[{\"x\":1},{\"x\":2}]",
|
||||
"",
|
||||
|
@ -5775,7 +5775,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"b",
|
||||
4L,
|
||||
3.3D,
|
||||
"true",
|
||||
1L,
|
||||
"1",
|
||||
0.0D,
|
||||
"{}",
|
||||
|
@ -5790,7 +5790,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[2.2,3.3,4.0]",
|
||||
null,
|
||||
"[\"a\",\"b\",\"c\"]",
|
||||
"[null,\"false\",\"true\"]",
|
||||
"[null,0,1]",
|
||||
"[[1,2],[3,4],[5,6,7]]",
|
||||
"[{\"x\":null},{\"x\":2}]",
|
||||
"",
|
||||
|
@ -5813,7 +5813,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"c",
|
||||
0L,
|
||||
4.4D,
|
||||
"true",
|
||||
1L,
|
||||
"hello",
|
||||
-1000.0D,
|
||||
"{}",
|
||||
|
@ -5828,7 +5828,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[1.1,2.2,3.3]",
|
||||
null,
|
||||
null,
|
||||
"[\"false\"]",
|
||||
"[0]",
|
||||
null,
|
||||
"[{\"x\":1000},{\"y\":2000}]",
|
||||
"",
|
||||
|
@ -5851,7 +5851,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"d",
|
||||
5L,
|
||||
5.9D,
|
||||
"false",
|
||||
0L,
|
||||
"",
|
||||
3.33D,
|
||||
"\"a\"",
|
||||
|
@ -5889,7 +5889,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"null",
|
||||
3L,
|
||||
2.0D,
|
||||
"",
|
||||
0L,
|
||||
"3.0",
|
||||
1.0D,
|
||||
"3.3",
|
||||
|
@ -5904,7 +5904,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[1.1,3.3]",
|
||||
"[null,2.2,null]",
|
||||
"[1,null,1]",
|
||||
"[\"true\",null,\"true\"]",
|
||||
"[1,null,1]",
|
||||
"[[1],null,[1,2,3]]",
|
||||
"[null,{\"x\":2}]",
|
||||
"",
|
||||
|
@ -5929,7 +5929,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
null,
|
||||
null,
|
||||
null,
|
||||
"true",
|
||||
1L,
|
||||
"51",
|
||||
-0.13D,
|
||||
"1",
|
||||
|
@ -5944,7 +5944,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
null,
|
||||
"[null]",
|
||||
null,
|
||||
"[\"true\",\"false\",\"true\"]",
|
||||
"[1,0,1]",
|
||||
null,
|
||||
"[{\"x\":1},{\"x\":2}]",
|
||||
null,
|
||||
|
@ -5967,7 +5967,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"",
|
||||
2L,
|
||||
null,
|
||||
"false",
|
||||
0L,
|
||||
"b",
|
||||
1.1D,
|
||||
"\"b\"",
|
||||
|
@ -5982,7 +5982,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[3.3,4.4,5.5]",
|
||||
"[999.0,null,5.5]",
|
||||
"[null,null,2.2]",
|
||||
"[\"true\",\"true\"]",
|
||||
"[1,1]",
|
||||
"[null,[null],[]]",
|
||||
"[{\"x\":3},{\"x\":4}]",
|
||||
null,
|
||||
|
@ -6005,7 +6005,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"a",
|
||||
1L,
|
||||
1.0D,
|
||||
"true",
|
||||
1L,
|
||||
"1",
|
||||
1.0D,
|
||||
"1",
|
||||
|
@ -6020,7 +6020,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[1.1,2.2,3.3]",
|
||||
"[1.1,2.2,null]",
|
||||
"[\"a\",\"1\",\"2.2\"]",
|
||||
"[\"true\",\"false\",\"true\"]",
|
||||
"[1,0,1]",
|
||||
"[[1,2,null],[3,4]]",
|
||||
"[{\"x\":1},{\"x\":2}]",
|
||||
null,
|
||||
|
@ -6043,7 +6043,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"b",
|
||||
4L,
|
||||
3.3D,
|
||||
"true",
|
||||
1L,
|
||||
"1",
|
||||
null,
|
||||
"{}",
|
||||
|
@ -6058,7 +6058,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[2.2,3.3,4.0]",
|
||||
null,
|
||||
"[\"a\",\"b\",\"c\"]",
|
||||
"[null,\"false\",\"true\"]",
|
||||
"[null,0,1]",
|
||||
"[[1,2],[3,4],[5,6,7]]",
|
||||
"[{\"x\":null},{\"x\":2}]",
|
||||
null,
|
||||
|
@ -6081,7 +6081,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"c",
|
||||
null,
|
||||
4.4D,
|
||||
"true",
|
||||
1L,
|
||||
"hello",
|
||||
-1000.0D,
|
||||
"{}",
|
||||
|
@ -6096,7 +6096,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[1.1,2.2,3.3]",
|
||||
null,
|
||||
null,
|
||||
"[\"false\"]",
|
||||
"[0]",
|
||||
null,
|
||||
"[{\"x\":1000},{\"y\":2000}]",
|
||||
null,
|
||||
|
@ -6119,7 +6119,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"d",
|
||||
5L,
|
||||
5.9D,
|
||||
"false",
|
||||
0L,
|
||||
null,
|
||||
3.33D,
|
||||
"\"a\"",
|
||||
|
@ -6172,7 +6172,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
"[1.1,3.3]",
|
||||
"[null,2.2,null]",
|
||||
"[1,null,1]",
|
||||
"[\"true\",null,\"true\"]",
|
||||
"[1,null,1]",
|
||||
"[[1],null,[1,2,3]]",
|
||||
"[null,{\"x\":2}]",
|
||||
null,
|
||||
|
@ -6196,7 +6196,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
.add("str", ColumnType.STRING)
|
||||
.add("long", ColumnType.LONG)
|
||||
.add("double", ColumnType.DOUBLE)
|
||||
.add("bool", ColumnType.STRING)
|
||||
.add("bool", ColumnType.LONG)
|
||||
.add("variant", ColumnType.STRING)
|
||||
.add("variantNumeric", ColumnType.DOUBLE)
|
||||
.add("variantEmptyObj", ColumnType.NESTED_DATA)
|
||||
|
@ -6211,7 +6211,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
|
|||
.add("arrayDouble", ColumnType.DOUBLE_ARRAY)
|
||||
.add("arrayDoubleNulls", ColumnType.DOUBLE_ARRAY)
|
||||
.add("arrayVariant", ColumnType.STRING_ARRAY)
|
||||
.add("arrayBool", ColumnType.STRING_ARRAY)
|
||||
.add("arrayBool", ColumnType.LONG_ARRAY)
|
||||
.add("arrayNestedLong", ColumnType.NESTED_DATA)
|
||||
.add("arrayObject", ColumnType.NESTED_DATA)
|
||||
.add("null", ColumnType.STRING)
|
||||
|
|
|
@ -29,7 +29,6 @@ import org.apache.druid.java.util.common.guava.Yielder;
|
|||
import org.apache.druid.java.util.common.guava.Yielders;
|
||||
import org.apache.druid.java.util.common.io.Closer;
|
||||
import org.apache.druid.java.util.common.logger.Logger;
|
||||
import org.apache.druid.math.expr.ExpressionProcessing;
|
||||
import org.apache.druid.query.QueryContexts;
|
||||
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
|
||||
import org.apache.druid.segment.QueryableIndex;
|
||||
|
@ -113,7 +112,6 @@ public class SqlVectorizedExpressionSanityTest extends InitializedNullHandlingTe
|
|||
@BeforeClass
|
||||
public static void setupClass()
|
||||
{
|
||||
ExpressionProcessing.initializeForStrictBooleansTests(true);
|
||||
CLOSER = Closer.create();
|
||||
|
||||
final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
|
||||
|
@ -162,7 +160,6 @@ public class SqlVectorizedExpressionSanityTest extends InitializedNullHandlingTe
|
|||
public static void teardownClass() throws IOException
|
||||
{
|
||||
CLOSER.close();
|
||||
ExpressionProcessing.initializeForTests();
|
||||
}
|
||||
|
||||
@Parameterized.Parameters(name = "query = {0}")
|
||||
|
|
Loading…
Reference in New Issue