allow expression "best effort" type determination to better handle mixed type arrays (#14438)

processing/src/main/java/org/apache/druid/math/expr/ExprEval.java

@@ -36,6 +36,7 @@ import javax.annotation.Nullable;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Generic result holder for evaluated {@link Expr} containing the value and {@link ExprType} of the value to allow
@@ -183,7 +184,7 @@ public abstract class ExprEval<T>
         for (Object o : val) {
           if (o != null) {
             ExprEval<?> eval = ExprEval.bestEffortOf(o);
-            elementType = ExpressionTypeConversion.coerceArrayTypes(elementType, eval.type());
+            elementType = ExpressionTypeConversion.leastRestrictiveType(elementType, eval.type());
             evals[i++] = eval;
           } else {
             evals[i++] = null;
@@ -506,6 +507,11 @@ public abstract class ExprEval<T>
       return new StringExprEval(StringUtils.encodeBase64String((byte[]) val));
     }
 
+
+    if (val instanceof Map) {
+      return ofComplex(ExpressionType.NESTED_DATA, val);
+    }
+
     // is this cool?
     return ofComplex(ExpressionType.UNKNOWN_COMPLEX, val);
   }

processing/src/main/java/org/apache/druid/math/expr/ExpressionTypeConversion.java

@@ -19,6 +19,7 @@
 
 package org.apache.druid.math.expr;
 
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.Types;
 
 import javax.annotation.Nullable;
@@ -151,8 +152,13 @@ public class ExpressionTypeConversion
     return numeric(type, other);
   }
 
+  /**
+   * {@link ColumnType#leastRestrictiveType(ColumnType, ColumnType)} but for expression
+   * types
+   */
   @Nullable
-  public static ExpressionType coerceArrayTypes(@Nullable ExpressionType type, @Nullable ExpressionType other)
+  public static ExpressionType leastRestrictiveType(@Nullable ExpressionType type, @Nullable ExpressionType other)
+      throws Types.IncompatibleTypeException
   {
     if (type == null) {
       return other;
@@ -160,23 +166,66 @@ public class ExpressionTypeConversion
     if (other == null) {
       return type;
     }
-
+    if (type.is(ExprType.COMPLEX) && other.is(ExprType.COMPLEX)) {
-    if (Objects.equals(type, other)) {
+      if (type.getComplexTypeName() == null) {
+        return other;
+      }
+      if (other.getComplexTypeName() == null) {
+        return type;
+      }
+      if (!Objects.equals(type, other)) {
+        throw new Types.IncompatibleTypeException(type, other);
+      }
       return type;
     }
-
+    // if either is nested data, use nested data, otherwise error
-    ExpressionType typeArrayType = type.isArray() ? type : ExpressionTypeFactory.getInstance().ofArray(type);
+    if (type.is(ExprType.COMPLEX) || other.is(ExprType.COMPLEX)) {
-    ExpressionType otherArrayType = other.isArray() ? other : ExpressionTypeFactory.getInstance().ofArray(other);
+      if (ExpressionType.NESTED_DATA.equals(type) || ExpressionType.NESTED_DATA.equals(other)) {
-
+        return ExpressionType.NESTED_DATA;
-    if (typeArrayType.getElementType().isPrimitive() && otherArrayType.getElementType().isPrimitive()) {
+      }
-      ExpressionType newElementType = function(
+      throw new Types.IncompatibleTypeException(type, other);
-          (ExpressionType) typeArrayType.getElementType(),
-          (ExpressionType) otherArrayType.getElementType()
-      );
-      return ExpressionTypeFactory.getInstance().ofArray(newElementType);
     }
 
-    throw new Types.IncompatibleTypeException(type, other);
+    // arrays convert based on least restrictive element type
+    if (type.isArray()) {
+      if (other.equals(type.getElementType())) {
+        return type;
+      }
+      final ExpressionType commonElementType;
+      // commonElementType cannot be null if we got this far, we always return a value unless both args are null
+      if (other.isArray()) {
+        commonElementType = leastRestrictiveType(
+            (ExpressionType) type.getElementType(),
+            (ExpressionType) other.getElementType()
+        );
+
+        return ExpressionTypeFactory.getInstance().ofArray(commonElementType);
+      } else {
+        commonElementType = leastRestrictiveType(
+            (ExpressionType) type.getElementType(),
+            other
+        );
+      }
+      return ExpressionTypeFactory.getInstance().ofArray(commonElementType);
+    }
+    if (other.isArray()) {
+      if (type.equals(type.getElementType())) {
+        return type;
+      }
+      final ExpressionType commonElementType;
+
+      commonElementType = leastRestrictiveType(
+          type,
+          (ExpressionType) other.getElementType()
+      );
+      return ExpressionTypeFactory.getInstance().ofArray(commonElementType);
+    }
+    // if either argument is a string, type becomes a string
+    if (Types.is(type, ExprType.STRING) || Types.is(other, ExprType.STRING)) {
+      return ExpressionType.STRING;
+    }
+
+    return numeric(type, other);
   }
 
 

processing/src/test/java/org/apache/druid/math/expr/EvalTest.java

@@ -22,6 +22,7 @@ package org.apache.druid.math.expr;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import junitparams.converters.Nullable;
+import org.apache.druid.collections.SerializablePair;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.IAE;
 import org.apache.druid.java.util.common.StringUtils;
@@ -1383,12 +1384,18 @@ public class EvalTest extends InitializedNullHandlingTest
         new Object[] {"1.0", "2", "3", "true", "false"}
     );
 
-    // best effort of doesn't know of nested type, what happens if we have some nested data?
     assertBestEffortOf(
         ImmutableMap.of("x", 1L, "y", 2L),
-        ExpressionType.UNKNOWN_COMPLEX,
+        ExpressionType.NESTED_DATA,
         ImmutableMap.of("x", 1L, "y", 2L)
     );
+
+    SerializablePair<String, Long> someOtherComplex = new SerializablePair<>("hello", 1234L);
+    assertBestEffortOf(
+        someOtherComplex,
+        ExpressionType.UNKNOWN_COMPLEX,
+        someOtherComplex
+    );
   }
 
   private void assertBestEffortOf(@Nullable Object val, ExpressionType expectedType, @Nullable Object expectedValue)

processing/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java

@@ -21,11 +21,13 @@ package org.apache.druid.math.expr;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import org.apache.druid.collections.SerializablePair;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.NonnullPair;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.segment.column.TypeStrategies;
 import org.apache.druid.segment.column.TypeStrategiesTest;
+import org.apache.druid.segment.column.Types;
 import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -342,35 +344,35 @@ public class ExprEvalTest extends InitializedNullHandlingTest
         coerced.rhs
     );
 
-    Map<String, Object> unknown1 = ImmutableMap.of("x", 1L, "y", 2L);
+    Map<String, Object> nested1 = ImmutableMap.of("x", 1L, "y", 2L);
-    Map<String, Object> unknown2 = ImmutableMap.of("x", 4L, "y", 5L);
+    Map<String, Object> nested2 = ImmutableMap.of("x", 4L, "y", 5L);
-    List<Map<String, Object>> listUnknownComplex = ImmutableList.of(unknown1, unknown2);
+    List<Map<String, Object>> listUnknownComplex = ImmutableList.of(nested1, nested2);
     coerced = ExprEval.coerceListToArray(listUnknownComplex, false);
-    Assert.assertEquals(ExpressionTypeFactory.getInstance().ofArray(ExpressionType.UNKNOWN_COMPLEX), coerced.lhs);
+    Assert.assertEquals(ExpressionTypeFactory.getInstance().ofArray(ExpressionType.NESTED_DATA), coerced.lhs);
     Assert.assertArrayEquals(
-        new Object[]{unknown1, unknown2},
+        new Object[]{nested1, nested2},
         coerced.rhs
     );
 
 
-    Map<String, Object> unknown3 = ImmutableMap.of("x", 5L, "y", 7L);
+    Map<String, Object> nested3 = ImmutableMap.of("x", 5L, "y", 7L);
-    Map<String, Object> unknown4 = ImmutableMap.of("x", 6L, "y", 8L);
+    Map<String, Object> nested4 = ImmutableMap.of("x", 6L, "y", 8L);
 
     List<List<Map<String, Object>>> nestedListsComplex = ImmutableList.of(
-        ImmutableList.of(unknown1, unknown2),
+        ImmutableList.of(nested1, nested2),
-        ImmutableList.of(unknown3, unknown4)
+        ImmutableList.of(nested3, nested4)
     );
     coerced = ExprEval.coerceListToArray(nestedListsComplex, false);
     Assert.assertEquals(
         ExpressionTypeFactory.getInstance().ofArray(
-            ExpressionTypeFactory.getInstance().ofArray(ExpressionType.UNKNOWN_COMPLEX)
+            ExpressionTypeFactory.getInstance().ofArray(ExpressionType.NESTED_DATA)
         ),
         coerced.lhs
     );
     Assert.assertArrayEquals(
         new Object[]{
-            new Object[]{unknown1, unknown2},
+            new Object[]{nested1, nested2},
-            new Object[]{unknown3, unknown4}
+            new Object[]{nested3, nested4}
         },
         coerced.rhs
     );
@@ -395,6 +397,41 @@ public class ExprEvalTest extends InitializedNullHandlingTest
         },
         coerced.rhs
     );
+
+    List<List<Object>> mixedNested = ImmutableList.of(
+        ImmutableList.of("a", "b", "c"),
+        ImmutableList.of(1L, 2L, 3L),
+        ImmutableList.of(3.0, 4.0, 5.0),
+        ImmutableList.of("a", 2L, 3.0, nested1)
+    );
+    coerced = ExprEval.coerceListToArray(mixedNested, false);
+    Assert.assertEquals(
+        ExpressionTypeFactory.getInstance().ofArray(
+            ExpressionTypeFactory.getInstance().ofArray(ExpressionType.NESTED_DATA)
+        ),
+        coerced.lhs
+    );
+    Assert.assertArrayEquals(
+        new Object[]{
+            new Object[]{"a", "b", "c"},
+            new Object[]{1L, 2L, 3L},
+            new Object[]{3.0, 4.0, 5.0},
+            new Object[]{"a", 2L, 3.0, nested1}
+        },
+        coerced.rhs
+    );
+
+    List<List<Object>> mixedUnknown = ImmutableList.of(
+        ImmutableList.of("a", "b", "c"),
+        ImmutableList.of(1L, 2L, 3L),
+        ImmutableList.of(3.0, 4.0, 5.0, new SerializablePair<>("hello", 1234L)),
+        ImmutableList.of("a", 2L, 3.0, nested1)
+    );
+    Throwable t = Assert.assertThrows(
+        Types.IncompatibleTypeException.class,
+        () -> ExprEval.coerceListToArray(mixedUnknown, false)
+    );
+    Assert.assertEquals("Cannot implicitly cast [DOUBLE] to [COMPLEX]", t.getMessage());
   }
 
   @Test

processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java

@@ -787,7 +787,7 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest
       Assert.assertEquals(
           "["
           + "[1672531200000, null, 0, 0.0, true, 51, 1, [], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [true, false, true], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
-          + "[1672531200000, null, 2, 0.0, false, b, b, 2, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
+          + "[1672531200000, null, 2, 0.0, false, b, b, 2, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
           + "[1672531200000, a, 1, 1.0, true, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [true, false, true], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
           + "[1672531200000, b, 4, 3.3, true, 4, {}, 4, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, false, true], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
           + "[1672531200000, c, 0, 4.4, true, hello, {}, [], {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [false], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
@@ -800,7 +800,7 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest
       Assert.assertEquals(
           "["
           + "[1672531200000, null, null, null, true, 51, 1, [], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [true, false, true], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
-          + "[1672531200000, , 2, null, false, b, b, 2, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
+          + "[1672531200000, , 2, null, false, b, b, 2, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
           + "[1672531200000, a, 1, 1.0, true, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [true, false, true], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
           + "[1672531200000, b, 4, 3.3, true, 4, {}, 4, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, false, true], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "
           + "[1672531200000, c, null, 4.4, true, hello, {}, [], {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [false], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], "

processing/src/test/resources/nested-all-types-test-data.json

@@ -1,5 +1,5 @@
 {"timestamp": "2023-01-01T00:00:00", "str":"a",    "long":1,     "double":1.0,  "bool": true,  "variant": 1,       "variantEmptyObj":1,   "variantEmtpyArray":1,  "obj":{"a": 100, "b": {"x": "a", "y": 1.1, "z": [1, 2, 3, 4]}},    "complexObj":{"x": 1234, "y": [{"l": ["a", "b", "c"], "m": "a", "n": 1},{"l": ["a", "b", "c"], "m": "a", "n": 1}], "z": {"a": [1.1, 2.2, 3.3], "b": true}},     "arrayString": ["a", "b"],       "arrayStringNulls": ["a", "b"],         "arrayLong":[1, 2, 3],    "arrayLongNulls":[1, null,3],   "arrayDouble":[1.1, 2.2, 3.3],  "arrayDoubleNulls":[1.1, 2.2, null],  "arrayVariant":["a", 1, 2.2],      "arrayBool":[true, false, true], "arrayNestedLong":[[1, 2, null], [3, 4]],      "arrayObject":[{"x": 1},{"x":2}],       "null": null, "cstr": "hello", "clong": 1234, "cdouble": 1.234, "cObj":{"x": 1, "y": "hello", "z": {"a": 1.1, "b": 1234, "c": ["a", "b", "c"]}}, "cstringArray": ["a", "b", "c"], "cLongArray": [1, 2, 3], "cDoubleArray": [1.1, 2.2, 3.3], "cEmptyArray":[], "cEmptyObj":{}, "cNullArray": [null, null], "cEmptyObjectArray": [{},{},{}], "cObjectArray": [{"a":"b", "x":1, "y":1.3}]}
-{"timestamp": "2023-01-01T00:00:00", "str":"",     "long":2,                    "bool": false, "variant": "b",     "variantEmptyObj":"b", "variantEmtpyArray":2,  "obj":{"a": 200, "b": {"x": "b", "y": 1.1, "z": [2, 4, 6]}},       "complexObj":{"x": 10,   "y": [{"l": ["b", "b", "c"], "m": "b", "n": 2}],                                          "z": {"a": [5.5], "b": false}},              "arrayString": ["a", "b", "c"],  "arrayStringNulls": [null, "b"],        "arrayLong":[2, 3],                                       "arrayDouble":[3.3, 4.4, 5.5],  "arrayDoubleNulls":[999, null, 5.5],  "arrayVariant":[null, null, 2.2],  "arrayBool":[true, true],        "arrayNestedLong":[null, [null], []],          "arrayObject":[{"x": 3},{"x":4}],       "null": null, "cstr": "hello", "clong": 1234, "cdouble": 1.234, "cObj":{"x": 1, "y": "hello", "z": {"a": 1.1, "b": 1234, "c": ["a", "b", "c"]}}, "cstringArray": ["a", "b", "c"], "cLongArray": [1, 2, 3], "cDoubleArray": [1.1, 2.2, 3.3], "cEmptyArray":[], "cEmptyObj":{}, "cNullArray": [null, null], "cEmptyObjectArray": [{},{},{}], "cObjectArray": [{"a":"b", "x":1, "y":1.3}]}
+{"timestamp": "2023-01-01T00:00:00", "str":"",     "long":2,                    "bool": false, "variant": "b",     "variantEmptyObj":"b", "variantEmtpyArray":2,  "obj":{"a": 200, "b": {"x": "b", "y": 1.1, "z": [2, 4, 6]}},       "complexObj":{"x": 10,   "y": [{"l": ["b", "b", "c"], "m": "b", "n": 2}, [1, 2, 3]],                               "z": {"a": [5.5], "b": false}},              "arrayString": ["a", "b", "c"],  "arrayStringNulls": [null, "b"],        "arrayLong":[2, 3],                                       "arrayDouble":[3.3, 4.4, 5.5],  "arrayDoubleNulls":[999, null, 5.5],  "arrayVariant":[null, null, 2.2],  "arrayBool":[true, true],        "arrayNestedLong":[null, [null], []],          "arrayObject":[{"x": 3},{"x":4}],       "null": null, "cstr": "hello", "clong": 1234, "cdouble": 1.234, "cObj":{"x": 1, "y": "hello", "z": {"a": 1.1, "b": 1234, "c": ["a", "b", "c"]}}, "cstringArray": ["a", "b", "c"], "cLongArray": [1, 2, 3], "cDoubleArray": [1.1, 2.2, 3.3], "cEmptyArray":[], "cEmptyObj":{}, "cNullArray": [null, null], "cEmptyObjectArray": [{},{},{}], "cObjectArray": [{"a":"b", "x":1, "y":1.3}]}
 {"timestamp": "2023-01-01T00:00:00", "str":"null", "long":3,     "double":2.0,                 "variant": 3.0,     "variantEmptyObj":3.3, "variantEmtpyArray":3,  "obj":{"a": 300},                                                  "complexObj":{"x": 4,    "y": [{"l": [], "m": 100, "n": 3},{"l": ["a"]}, {"l": ["b"], "n": []}],                   "z": {"a": [], "b": true}},                  "arrayString": ["b", "c"],       "arrayStringNulls": ["d", null, "b"],   "arrayLong":[1, 2, 3, 4], "arrayLongNulls":[1, 2, 3],     "arrayDouble":[1.1, 3.3],       "arrayDoubleNulls":[null, 2.2, null], "arrayVariant":[1, null, 1],       "arrayBool":[true, null, true],  "arrayNestedLong":[[1], null, [1, 2, 3]],      "arrayObject":[null,{"x":2}],                         "cstr": "hello", "clong": 1234, "cdouble": 1.234, "cObj":{"x": 1, "y": "hello", "z": {"a": 1.1, "b": 1234, "c": ["a", "b", "c"]}}, "cstringArray": ["a", "b", "c"], "cLongArray": [1, 2, 3], "cDoubleArray": [1.1, 2.2, 3.3], "cEmptyArray":[], "cEmptyObj":{}, "cNullArray": [null, null], "cEmptyObjectArray": [{},{},{}], "cObjectArray": [{"a":"b", "x":1, "y":1.3}]}
 {"timestamp": "2023-01-01T00:00:00", "str":"b",    "long":4,     "double":3.3,  "bool": true,  "variant": "4",     "variantEmptyObj":{},  "variantEmtpyArray":4,  "obj":{"a": 400, "b": {"x": "d", "y": 1.1, "z": [3, 4]}},          "complexObj":{"x": 1234,                                                                                           "z": {"a": [1.1, 2.2, 3.3], "b": true}},     "arrayString": ["d", "e"],       "arrayStringNulls": ["b", "b"],         "arrayLong":[1, 4],       "arrayLongNulls":[1],           "arrayDouble":[2.2, 3.3, 4.0],                                        "arrayVariant":["a", "b", "c"],    "arrayBool":[null, false, true], "arrayNestedLong":[[1, 2], [3, 4], [5, 6, 7]], "arrayObject":[{"x": null},{"x":2}],    "null": null, "cstr": "hello", "clong": 1234, "cdouble": 1.234, "cObj":{"x": 1, "y": "hello", "z": {"a": 1.1, "b": 1234, "c": ["a", "b", "c"]}}, "cstringArray": ["a", "b", "c"], "cLongArray": [1, 2, 3], "cDoubleArray": [1.1, 2.2, 3.3], "cEmptyArray":[], "cEmptyObj":{}, "cNullArray": [null, null], "cEmptyObjectArray": [{},{},{}], "cObjectArray": [{"a":"b", "x":1, "y":1.3}]}
 {"timestamp": "2023-01-01T00:00:00", "str":"c",    "long": null, "double":4.4,  "bool": true,  "variant": "hello", "variantEmptyObj":{},  "variantEmtpyArray":[], "obj":{"a": 500, "b": {"x": "e", "z": [1, 2, 3, 4]}},              "complexObj":{"x": 11,   "y": [],                                                                                  "z": {"a": [null], "b": false}},             "arrayString": null,                                                     "arrayLong":[1, 2, 3],    "arrayLongNulls":[],            "arrayDouble":[1.1, 2.2, 3.3],  "arrayDoubleNulls":null,                                                 "arrayBool":[false],                                                            "arrayObject":[{"x": 1000},{"y":2000}], "null": null, "cstr": "hello", "clong": 1234, "cdouble": 1.234, "cObj":{"x": 1, "y": "hello", "z": {"a": 1.1, "b": 1234, "c": ["a", "b", "c"]}}, "cstringArray": ["a", "b", "c"], "cLongArray": [1, 2, 3], "cDoubleArray": [1.1, 2.2, 3.3], "cEmptyArray":[], "cEmptyObj":{}, "cNullArray": [null, null], "cEmptyObjectArray": [{},{},{}], "cObjectArray": [{"a":"b", "x":1, "y":1.3}]}

sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java

@@ -5229,7 +5229,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
         useDefault ?
         ImmutableList.of(
             new Object[]{1672531200000L, "", 0L, 0.0D, "true", "51", "1", "[]", "{\"a\":700,\"b\":{\"x\":\"g\",\"y\":1.1,\"z\":[9,null,9,9]}}", "{\"x\":400,\"y\":[{\"l\":[null],\"m\":100,\"n\":5},{\"l\":[\"a\",\"b\",\"c\"],\"m\":\"a\",\"n\":1}],\"z\":{}}", null, "[\"a\",\"b\"]", null, "[2,3]", null, "[null]", null, "[\"true\",\"false\",\"true\"]", null, "[{\"x\":1},{\"x\":2}]", "", "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
-            new Object[]{1672531200000L, "", 2L, 0.0D, "false", "b", "\"b\"", "2", "{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]}}", "{\"x\":10,\"y\":[{\"l\":[\"b\",\"b\",\"c\"],\"m\":\"b\",\"n\":2}],\"z\":{\"a\":[5.5],\"b\":false}}", "[\"a\",\"b\",\"c\"]", "[null,\"b\"]", "[2,3]", null, "[3.3,4.4,5.5]", "[999.0,null,5.5]", "[null,null,2.2]", "[\"true\",\"true\"]", "[null,[null],[]]", "[{\"x\":3},{\"x\":4}]", "", "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
+            new Object[]{1672531200000L, "", 2L, 0.0D, "false", "b", "\"b\"", "2", "{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]}}", "{\"x\":10,\"y\":[{\"l\":[\"b\",\"b\",\"c\"],\"m\":\"b\",\"n\":2},[1,2,3]],\"z\":{\"a\":[5.5],\"b\":false}}", "[\"a\",\"b\",\"c\"]", "[null,\"b\"]", "[2,3]", null, "[3.3,4.4,5.5]", "[999.0,null,5.5]", "[null,null,2.2]", "[\"true\",\"true\"]", "[null,[null],[]]", "[{\"x\":3},{\"x\":4}]", "", "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
             new Object[]{1672531200000L, "a", 1L, 1.0D, "true", "1", "1", "1", "{\"a\":100,\"b\":{\"x\":\"a\",\"y\":1.1,\"z\":[1,2,3,4]}}", "{\"x\":1234,\"y\":[{\"l\":[\"a\",\"b\",\"c\"],\"m\":\"a\",\"n\":1},{\"l\":[\"a\",\"b\",\"c\"],\"m\":\"a\",\"n\":1}],\"z\":{\"a\":[1.1,2.2,3.3],\"b\":true}}", "[\"a\",\"b\"]", "[\"a\",\"b\"]", "[1,2,3]", "[1,null,3]", "[1.1,2.2,3.3]", "[1.1,2.2,null]", "[\"a\",\"1\",\"2.2\"]", "[\"true\",\"false\",\"true\"]", "[[1,2,null],[3,4]]", "[{\"x\":1},{\"x\":2}]", "", "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
             new Object[]{1672531200000L, "b", 4L, 3.3D, "true", "4", "{}", "4", "{\"a\":400,\"b\":{\"x\":\"d\",\"y\":1.1,\"z\":[3,4]}}", "{\"x\":1234,\"z\":{\"a\":[1.1,2.2,3.3],\"b\":true}}", "[\"d\",\"e\"]", "[\"b\",\"b\"]", "[1,4]", "[1]", "[2.2,3.3,4.0]", null, "[\"a\",\"b\",\"c\"]", "[null,\"false\",\"true\"]", "[[1,2],[3,4],[5,6,7]]", "[{\"x\":null},{\"x\":2}]", "", "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
             new Object[]{1672531200000L, "c", 0L, 4.4D, "true", "hello", "{}", "[]", "{\"a\":500,\"b\":{\"x\":\"e\",\"z\":[1,2,3,4]}}", "{\"x\":11,\"y\":[],\"z\":{\"a\":[null],\"b\":false}}", null, null, "[1,2,3]", "[]", "[1.1,2.2,3.3]", null, null, "[\"false\"]", null, "[{\"x\":1000},{\"y\":2000}]", "", "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
@@ -5238,7 +5238,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
         ) :
         ImmutableList.of(
             new Object[]{1672531200000L, null, null, null, "true", "51", "1", "[]", "{\"a\":700,\"b\":{\"x\":\"g\",\"y\":1.1,\"z\":[9,null,9,9]}}", "{\"x\":400,\"y\":[{\"l\":[null],\"m\":100,\"n\":5},{\"l\":[\"a\",\"b\",\"c\"],\"m\":\"a\",\"n\":1}],\"z\":{}}", null, "[\"a\",\"b\"]", null, "[2,3]", null, "[null]", null, "[\"true\",\"false\",\"true\"]", null, "[{\"x\":1},{\"x\":2}]", null, "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
-            new Object[]{1672531200000L, "", 2L, null, "false", "b", "\"b\"", "2", "{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]}}", "{\"x\":10,\"y\":[{\"l\":[\"b\",\"b\",\"c\"],\"m\":\"b\",\"n\":2}],\"z\":{\"a\":[5.5],\"b\":false}}", "[\"a\",\"b\",\"c\"]", "[null,\"b\"]", "[2,3]", null, "[3.3,4.4,5.5]", "[999.0,null,5.5]", "[null,null,2.2]", "[\"true\",\"true\"]", "[null,[null],[]]", "[{\"x\":3},{\"x\":4}]", null, "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
+            new Object[]{1672531200000L, "", 2L, null, "false", "b", "\"b\"", "2", "{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]}}", "{\"x\":10,\"y\":[{\"l\":[\"b\",\"b\",\"c\"],\"m\":\"b\",\"n\":2},[1,2,3]],\"z\":{\"a\":[5.5],\"b\":false}}", "[\"a\",\"b\",\"c\"]", "[null,\"b\"]", "[2,3]", null, "[3.3,4.4,5.5]", "[999.0,null,5.5]", "[null,null,2.2]", "[\"true\",\"true\"]", "[null,[null],[]]", "[{\"x\":3},{\"x\":4}]", null, "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
             new Object[]{1672531200000L, "a", 1L, 1.0D, "true", "1", "1", "1", "{\"a\":100,\"b\":{\"x\":\"a\",\"y\":1.1,\"z\":[1,2,3,4]}}", "{\"x\":1234,\"y\":[{\"l\":[\"a\",\"b\",\"c\"],\"m\":\"a\",\"n\":1},{\"l\":[\"a\",\"b\",\"c\"],\"m\":\"a\",\"n\":1}],\"z\":{\"a\":[1.1,2.2,3.3],\"b\":true}}", "[\"a\",\"b\"]", "[\"a\",\"b\"]", "[1,2,3]", "[1,null,3]", "[1.1,2.2,3.3]", "[1.1,2.2,null]", "[\"a\",\"1\",\"2.2\"]", "[\"true\",\"false\",\"true\"]", "[[1,2,null],[3,4]]", "[{\"x\":1},{\"x\":2}]", null, "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
             new Object[]{1672531200000L, "b", 4L, 3.3D, "true", "4", "{}", "4", "{\"a\":400,\"b\":{\"x\":\"d\",\"y\":1.1,\"z\":[3,4]}}", "{\"x\":1234,\"z\":{\"a\":[1.1,2.2,3.3],\"b\":true}}", "[\"d\",\"e\"]", "[\"b\",\"b\"]", "[1,4]", "[1]", "[2.2,3.3,4.0]", null, "[\"a\",\"b\",\"c\"]", "[null,\"false\",\"true\"]", "[[1,2],[3,4],[5,6,7]]", "[{\"x\":null},{\"x\":2}]", null, "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},
             new Object[]{1672531200000L, "c", null, 4.4D, "true", "hello", "{}", "[]", "{\"a\":500,\"b\":{\"x\":\"e\",\"z\":[1,2,3,4]}}", "{\"x\":11,\"y\":[],\"z\":{\"a\":[null],\"b\":false}}", null, null, "[1,2,3]", "[]", "[1.1,2.2,3.3]", null, null, "[\"false\"]", null, "[{\"x\":1000},{\"y\":2000}]", null, "hello", 1234L, 1.234D, "{\"x\":1,\"y\":\"hello\",\"z\":{\"a\":1.1,\"b\":1234,\"c\":[\"a\",\"b\",\"c\"]}}", "[\"a\",\"b\",\"c\"]", "[1,2,3]", "[1.1,2.2,3.3]", "[]", "{}", "[null,null]", "[{},{},{}]", "[{\"a\":\"b\",\"x\":1,\"y\":1.3}]", 1L},