From acadc2df20117a3c5aac836500ef0b11880990ea Mon Sep 17 00:00:00 2001 From: Abhishek Radhakrishnan Date: Tue, 13 Aug 2024 07:50:03 -0700 Subject: [PATCH] Handle Delta StructType, ArrayType and MapType (#16884) Handle the following Delta complex types: a. StructType as JSON b. ArrayType as Java list c. MapType as Java map Generate and add a new Delta table complex-types-table that contains the above complex types for testing. Update the tests to include a parameterized test with complex-types-table, with the expectations defined in ComplexTypesDeltaTable.java. --- .../druid/delta/input/DeltaInputRow.java | 14 ++ .../delta/input/ComplexTypesDeltaTable.java | 135 ++++++++++++++++++ .../druid/delta/input/DeltaInputRowTest.java | 5 +- .../delta/input/DeltaInputSourceTest.java | 5 + .../src/test/resources/README.md | 11 ++ ...-9b74-0cc7b853103a-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...-834e-2a1cb6601eb8-c000.snappy.parquet.crc | Bin 0 -> 36 bytes ...-82a4-f3f273308e53-c000.snappy.parquet.crc | Bin 0 -> 36 bytes ...-bb38-f627c47eb20b-c000.snappy.parquet.crc | Bin 0 -> 36 bytes ...-bc77-0106e7f28f7a-c000.snappy.parquet.crc | Bin 0 -> 36 bytes ...-b605-506c942cd969-c000.snappy.parquet.crc | Bin 0 -> 36 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 52 bytes .../_delta_log/00000000000000000000.json | 8 ++ ...4a53-9b74-0cc7b853103a-c000.snappy.parquet | Bin 0 -> 1488 bytes ...4e91-834e-2a1cb6601eb8-c000.snappy.parquet | Bin 0 -> 3288 bytes ...4ef3-82a4-f3f273308e53-c000.snappy.parquet | Bin 0 -> 3291 bytes ...404a-bb38-f627c47eb20b-c000.snappy.parquet | Bin 0 -> 3289 bytes ...4141-bc77-0106e7f28f7a-c000.snappy.parquet | Bin 0 -> 3290 bytes ...43fe-b605-506c942cd969-c000.snappy.parquet | Bin 0 -> 3291 bytes .../src/test/resources/create_delta_table.py | 63 +++++++- 20 files changed, 235 insertions(+), 6 deletions(-) create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet create mode 100644 extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet diff --git a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java index acf909452a0..442412dd154 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java +++ b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java @@ -19,6 +19,10 @@ package org.apache.druid.delta.input; +import io.delta.kernel.data.ArrayValue; +import io.delta.kernel.data.MapValue; +import io.delta.kernel.internal.util.VectorUtils; +import io.delta.kernel.types.ArrayType; import io.delta.kernel.types.BinaryType; import io.delta.kernel.types.BooleanType; import io.delta.kernel.types.ByteType; @@ -29,6 +33,7 @@ import io.delta.kernel.types.DoubleType; import io.delta.kernel.types.FloatType; import io.delta.kernel.types.IntegerType; import io.delta.kernel.types.LongType; +import io.delta.kernel.types.MapType; import io.delta.kernel.types.ShortType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructField; @@ -197,6 +202,15 @@ public class DeltaInputRow implements InputRow return String.valueOf(charArray); } else if (dataType instanceof DecimalType) { return dataRow.getDecimal(columnOrdinal).longValue(); + } else if (dataType instanceof StructType) { + final io.delta.kernel.data.Row structRow = dataRow.getStruct(columnOrdinal); + return RowSerde.convertRowToJsonObject(structRow); + } else if (dataType instanceof ArrayType) { + final ArrayValue arrayRow = dataRow.getArray(columnOrdinal); + return VectorUtils.toJavaList(arrayRow); + } else if (dataType instanceof MapType) { + final MapValue map = dataRow.getMap(columnOrdinal); + return VectorUtils.toJavaMap(map); } else { throw InvalidInput.exception( "Unsupported data type[%s] for fieldName[%s].", diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java new file mode 100644 index 00000000000..7fdffc03041 --- /dev/null +++ b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.delta.input; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.data.input.ColumnsFilter; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.segment.AutoTypeColumnSchema; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Refer to extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md to generate the + * sample complex types Delta Lake table used in the unit tests. + * + */ +public class ComplexTypesDeltaTable +{ + /** + * The Delta table path used by unit tests. + */ + public static final String DELTA_TABLE_PATH = "src/test/resources/complex-types-table"; + + /** + * The list of dimensions in the Delta table {@link #DELTA_TABLE_PATH}. + */ + public static final List DIMENSIONS = ImmutableList.of( + "id", + "array_info", + "struct_info", + "nested_struct_info", + "map_info" + ); + + /** + * The expected set of rows from the first checkpoint file {@code {@link #DELTA_TABLE_PATH}/_delta_log/00000000000000000000.json} + */ + private static final List> SPLIT_0_EXPECTED_ROWS = new ArrayList<>( + ImmutableList.of( + ImmutableMap.of( + "id", 0L, + "array_info", ImmutableList.of(0, 1, 2, 3), + "struct_info", ImmutableMap.of("id", 0L, "name", "0"), + "nested_struct_info", ImmutableMap.of("id", 0L, "name", "0", "nested", ImmutableMap.of("nested_int", 0, "nested_double", 1.0)), + "map_info", ImmutableMap.of("key1", 1.0f, "key2", 1.0f) + ), + ImmutableMap.of( + "id", 1L, + "array_info", ImmutableList.of(1, 2, 3, 4), + "struct_info", ImmutableMap.of("id", 1L, "name", "1"), + "nested_struct_info", ImmutableMap.of("id", 1L, "name", "1", "nested", ImmutableMap.of("nested_int", 1, "nested_double", 2.0)), + "map_info", ImmutableMap.of("key1", 2.0f, "key2", 2.0f) + ), + ImmutableMap.of( + "id", 2L, + "array_info", ImmutableList.of(2, 3, 4, 5), + "struct_info", ImmutableMap.of("id", 2L, "name", "2"), + "nested_struct_info", ImmutableMap.of("id", 2L, "name", "2", "nested", ImmutableMap.of("nested_int", 2, "nested_double", 3.0)), + "map_info", ImmutableMap.of("key1", 3.0f, "key2", 3.0f) + ), + ImmutableMap.of( + "id", 3L, + "array_info", ImmutableList.of(3, 4, 5, 6), + "struct_info", ImmutableMap.of("id", 3L, "name", "3"), + "nested_struct_info", ImmutableMap.of("id", 3L, "name", "3", "nested", ImmutableMap.of("nested_int", 3, "nested_double", 4.0)), + "map_info", ImmutableMap.of("key1", 4.0f, "key2", 4.0f) + ), + ImmutableMap.of( + "id", 4L, + "array_info", ImmutableList.of(4, 5, 6, 7), + "struct_info", ImmutableMap.of("id", 4L, "name", "4"), + "nested_struct_info", ImmutableMap.of("id", 4L, "name", "4", "nested", ImmutableMap.of("nested_int", 4, "nested_double", 5.0)), + "map_info", ImmutableMap.of("key1", 5.0f, "key2", 5.0f) + ) + ) + ); + + /** + * Mapping of checkpoint file identifier to the list of expected rows in that checkpoint. + */ + public static final Map>> SPLIT_TO_EXPECTED_ROWS = new HashMap<>( + ImmutableMap.of( + 0, SPLIT_0_EXPECTED_ROWS + ) + ); + + /** + * Complete set of expected rows across all checkpoint files for {@link #DELTA_TABLE_PATH}. + */ + public static final List> EXPECTED_ROWS = SPLIT_TO_EXPECTED_ROWS.values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + + /** + * The Druid schema used for ingestion of {@link #DELTA_TABLE_PATH}. + */ + public static final InputRowSchema FULL_SCHEMA = new InputRowSchema( + new TimestampSpec("na", "posix", DateTimes.of("2024-01-01")), + new DimensionsSpec( + ImmutableList.of( + new AutoTypeColumnSchema("id", null), + new AutoTypeColumnSchema("array_info", null), + new AutoTypeColumnSchema("struct_info", null), + new AutoTypeColumnSchema("nested_struct_info", null), + new AutoTypeColumnSchema("map_info", null) + ) + ), + ColumnsFilter.all() + ); +} diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java index 4e1c2566f02..4c1b57c434a 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java +++ b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java @@ -54,7 +54,8 @@ public class DeltaInputRowTest { Object[][] data = new Object[][]{ {NonPartitionedDeltaTable.DELTA_TABLE_PATH, NonPartitionedDeltaTable.FULL_SCHEMA, NonPartitionedDeltaTable.DIMENSIONS, NonPartitionedDeltaTable.EXPECTED_ROWS}, - {PartitionedDeltaTable.DELTA_TABLE_PATH, PartitionedDeltaTable.FULL_SCHEMA, PartitionedDeltaTable.DIMENSIONS, PartitionedDeltaTable.EXPECTED_ROWS} + {PartitionedDeltaTable.DELTA_TABLE_PATH, PartitionedDeltaTable.FULL_SCHEMA, PartitionedDeltaTable.DIMENSIONS, PartitionedDeltaTable.EXPECTED_ROWS}, + {ComplexTypesDeltaTable.DELTA_TABLE_PATH, ComplexTypesDeltaTable.FULL_SCHEMA, ComplexTypesDeltaTable.DIMENSIONS, ComplexTypesDeltaTable.EXPECTED_ROWS} }; return Arrays.asList(data); } @@ -116,7 +117,7 @@ public class DeltaInputRowTest } } } - Assert.assertEquals(NonPartitionedDeltaTable.EXPECTED_ROWS.size(), totalRecordCount); + Assert.assertEquals(expectedRows.size(), totalRecordCount); } @MethodSource("data") diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java index 3fe42676498..e6bcf9f5fc8 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java +++ b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java @@ -84,6 +84,11 @@ public class DeltaInputSourceTest PartitionedDeltaTable.DELTA_TABLE_PATH, PartitionedDeltaTable.FULL_SCHEMA, PartitionedDeltaTable.EXPECTED_ROWS + }, + { + ComplexTypesDeltaTable.DELTA_TABLE_PATH, + ComplexTypesDeltaTable.FULL_SCHEMA, + ComplexTypesDeltaTable.EXPECTED_ROWS } }; } diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md b/extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md index f1ac54fb8b3..f45b33ab62c 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md +++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md @@ -84,3 +84,14 @@ python3 create_delta_table.py --save_path=employee-delta-table-partitioned-name The resulting Delta table is checked in to the repo. The expectated rows to be used in tests are updated in `PartitionedDeltaTable.java` accordingly. + +### Complex types table `complex-types-table`: + +The test data in `resources/complex-types-table` contains 5 Delta records generated with 1 snapshot. +The table was generated by running the following commands: +```shell +python3 create_delta_table.py --save_path=complex-types-table --num_records=5 --gen_complex_types=True +``` + +The resulting Delta table is checked in to the repo. The expectated rows to be used in tests are updated in +`ComplexTypesDeltaTable.java` accordingly. diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a56f68f447baee7e6115aad6e4ccb68586175fde GIT binary patch literal 20 bcmYc;N@ieSU}8vExl-+maj;$ag?ZBfJy!=u literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..6b7e86bcf54a017d8701a0f47f79e34e0d195fa8 GIT binary patch literal 36 scmYc;N@ieSU}AWpcyWrDl6iFUzYoe4`yQ$3YabQ){QJ_}xX|U#0P?jD@c;k- literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..88b089e950346e420d746c1643b8066e824805ec GIT binary patch literal 36 scmYc;N@ieSU}A9Wb6vu1GAH@3#S$0$)Uyw=TO66wjXf^h3)8y@0NEi8w*UYD literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..7f4972520056438a8fcb3fe998edfe7d90c9c2ba GIT binary patch literal 36 scmYc;N@ieSU}9Ju*|hMoT={`_+Ix=)+>sN})jm_CH|1-cb6)o~0QV*jApigX literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..cd8fc7a087f660653d9af8d14911dbbe31a7a752 GIT binary patch literal 36 scmYc;N@ieSU}8|*n37}aTBY}wL1<<+%e~pmEFZ4vvu|kVyLzV@0Ll3cd;kCd literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..038082a933b714e530c2c4cea6b49f5b391f4589 GIT binary patch literal 36 scmYc;N@ieSU}A{lNPTX?c*${h!bFyh?NT>Z1^zgjZtQX4UYOoR0N*?gv;Y7A literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..311d2a22b04d925d5e2f9cfdb60e405bbaafd525 GIT binary patch literal 52 zcmV-40L%Yla$^7h00ICC87fl@eX@`q3>oWVEG+?7lFEsb7v-zj%vt*E#AO1vFwe6v KP0fF9Q|IN#5*5_| literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json new file mode 100644 index 00000000000..84803f9483c --- /dev/null +++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json @@ -0,0 +1,8 @@ +{"commitInfo":{"timestamp":1723511561738,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"17937"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"b9eae5f4-d55b-4c38-b365-8228ec09248e"}} +{"metaData":{"id":"ce998219-9bde-4831-b78c-14b11f919fbe","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_info\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"nested_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_info\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"float\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1723511559184}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet","partitionValues":{},"size":3288,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"maxValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"maxValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet","partitionValues":{},"size":3289,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"maxValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet","partitionValues":{},"size":3290,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"maxValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"maxValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7a105c0f74fff6b494898d601cdbf192592f6b05 GIT binary patch literal 1488 zcmd5+&1%~~5MET8%8_){fmJE6I^oDaQDX z`WShSzDN%p{a8vd*v_F)&_(m}eKX(8?DhLEF16af&px)|rHF2)vG*N(L7Rlux)!sP zBtwfeMHGqER0Z>pTC_2_B7_WUtfp0Ls70uSp(mihhrp!qFJcoVp2-tx6%=h@U@Z_n zbjjp(3SzOd0Ad4j`Z%SrKc$hUP()6dn+1YGa@s`0%fcYTWTqh{IrYVITfQ~v<>f!W zh1K+4=mb)vR+q46XelbArwCNfyyO*Wx7t0%!f0WOr9g%3Of35zqF99itM~1p?GSdp z6A|BO8+E-)z}Qjv!}h@lDU4vhPS)J4mcYk6@0|DeT!EG`AAei(R3mgLiRL0qhrt5> zf<$Y9ZSgT`3_ZR@?f64ol%*_tED({@BTWy3MW5B{ND?6 za4ZJ{Dgm|^U)E5rKYGxJr(P_6xiZ~jV!tta4_niv^XC&oJoVS1CYq{qm4|PJW H@h$%YG+oZJ literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bb2fb67389d2373731fc431f3a626e6744c61f53 GIT binary patch literal 3288 zcmd5>Ay~vkN-3p?^dd`{W?tK{lcbwW zR!b=qU5ZFm^rCnw*ke!X_8?f)lLxQ%uotC=J;vfe7L;AT_hvHlN<-82515Ae^S$r= ze&73j-}_!V_ws8NMmU2<@i+IMymKLgC3sb^iqORg2_Y1Z!G(XoUWUK#Kiv6E$zYYc zDF{8Uakq={m?A$xu~?o>;3l|((+k+ZZ-4yo83B2u6o;`mbMimnra=sU{PNMgVZkq* z5BwNp;a|og_&NO3C+|O&1;XJRCuEl}_Q%_wf0q<|g5Z50gpMa6t}?FR<1gL?^<;Qc za3qTavfNA|8Srk?H7mGZ1wR$1@x5Q~@1`ptIoRbiLYoYX3AS^zxz503A8e$=1(kSo zBOsmrtfXs@BmW~jVo4_m@nOb=bWLWH1oMDdv4XC}3je^rN6Xkx!yHeB!Gdx^uuL5Y z7H(HXGt&gjTpHgyLsUEqYeag6%;S~d=^z#wwad1)%$UbS1d7uwYoTDNfuVR=l_q?&_@l9sYtEqiUD)>vvn7L}O_#8s3v z*{!ubEW!H0b%?sOP8%NcA8vcCPSx`#$PpAygQMKA>vRmwWEhB~U24#_N1X+6igNZ8 z#h(J@#F-K6dd}$`{ho6KjEZnu-4rNH`F{H)85=4eo z07~#tL?k__IFHJSS#G zT*N`_Z?59KVNA$_A-0NbDTC9#Kqq*3!ns_pl4Rfv>T$wTEb zbltqBHCxN3y=p^4GE}XFIlM zYd;UI8{XO~HOhwH=#0FvRHLrbHp*|T8>}V5P|X2z;N~_P%kb9dxULOd%qT-krFo-= zXq4ACHuB-js6mGaAt4_FbI_n)_rrj0NtbC0?A2z&vulm^EKvD08|UT~y7p<}|74|H zs!%1Ms14TX*7+B7`ZW3+NYuvO( zbBmpt>x?>!6IN;5D%^02i`T~|Xtj7_yx^3s7pjg;t!oyYD3ly$(Y~VZ!H=UezUa#& JLUZsx&0n-Y%g_J- literal 0 HcmV?d00001 diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..641396cb6f7cf670208d1da2d475f6dcc17d490a GIT binary patch literal 3291 zcmd5y^BTX~+z_MJ;B*!U1*hR#u`IQCu2|F~*oud?~@Ky=z;^N^+!C zT4M~k6iV>Hg`P`Vs3|6=6j~@gxU|srQc5ocUlMZiq0k=GTX6DbX0fue`8cVN{rgWlnH zk(EfgtqKOP@7bkEy|wB#Z_l?DSNxhwvWO_d*uYf}m9UX@n@x9ZzFJ>uKo-^M3fNUr z3?-0UGwpDL71O`EX?0^gSO9*^kHPAK@wR; zGyv-3tB7JVXFkkpP^xB0L#oN*LNLX2iq7PZZ9h@z3Wq!KA)6G|V(3X&&8fRDn&SEA&==t|Cg^-~|XQyzs?LMt{c8AN3O(<@Zvt;UjWp zx=u{fS= z(4ugUoKPdO@%sRILa=+MV3*PDY0UVYbYlDobVq&INuT1wZw&O1`_KVUEFw}q0W_*( zseXBsZqUhm2N-yUlRUFdb3D%njO{^&<#|`hk!(-4qlbeNvI5vuAcnxeo{40{Vkm^N zR8H9;GvJpDi{U0*Yr3f8_`oHHj2$kZQ|J<10oBS?s9n|vc{g-q$WL^D+{`kcw&6zA z%wYwN#tD$(d~(na=)DwZ=izEKcpnwtV81j&W`W-}(6XSz@Kd``P~Ii}IN6WGv3@3j5UZZ_VxYkuf@ zZs;11``4{-ZPmBNEz#(#g0)oj1FvO`zqf9&mIzB#55$pL(5NrNtlka+7rK}=4lUI$ zSY1SGe0^i15bsPHbc_%a@-+}g4f8e;u~84*YP~fBRDx#X%)C<9K2H3f zc*;W+M+qpZz2kKA><%5jjsC}eCHWU-pUc$b){M7VNl~1!?vC8F368jVVSXx1&$RSZ zNCMEk;A0!&@`n`=LZ^ueKQ0-5xktjKpy7s~#$|7v`#{{>K*m&b@L#JnhoVj~zI7=M{e#xX{v%f81C@X*xoG`ts*Ti8Ph)s!)}Xp-G7l5|8mu{=hy(i}b7CJKqncsVdwQ zLf+7X+fY2F$bXW=U>2@2eiPo3H>p8iescUojUW#f0!Z?r>jByc}Pk6ld%Wg8_<44aO`J+w?VM?^~WCyy)f73;R zE_aebk$cWYo-2dzarmh;MSs}-W^b?r&0&{QgiLcVzRNFu+ur10GI9Zthl@gFNklgc z?ey0qT~oXIMTS(;86jd?=F_?+i%f*(9;;&Gx)vLcz+Bb8OUv1#FnGnIP9xavFfuK3 zbr7r&Ckz(4Ok;)k^1ahcrSsS$(bH^^mi^IDj4t@YU5U>6!xJg69>fD{Cr)R4heJ7V z*h$be-@31O3}F1@+DxjoZZ+>MwN}=gnnja@%A=AdRcum_w4~K+S{qB1dbxotDsv^o zRgg8gT4}jd!v5jf=Tx1VQ+K)lK+A2mS6pv_9753)9Ob%Ib4JNrnuAE%wYt-C9eXLd zM7em0;w?cr3Ff4aPysnjf??1%*p2pb)!`&+&06njDAD4u z_>oZ+KnXERn53um#~B=MSudzW(YY>EViKEW^BJ`imS{L)BpZ}eXhk+r`Usm4_B=_i zvyMXFCYePXJbe_v#fZ{E-L9%&Q`s6TXBJCdscWJD!6FLZdyAsKpM9iWUo>*KrFmC`ij33Q4_BwWb#9-D<%2`3Lfh+TX~qu%FepQ_X# znG>M-a%az}k!bKIfILU+-YGcbbVn)={vaL(e-7R0JsiX@*uyageI@qL0jT6DQalIL zuj5Gl3YG4#nM?-^BEw;xd8av>=jWX5vwoiELpe>8J=snl4nfEZ;82E!V8EkJha;}x zS!*<9uXq4Y`?-cYxL8wO9R~+4JY@WEflg*+wg%VIP1G*$gQ6SeqmX+kK<*^@p7s@7 z!uqI;qj3(TcRmH^$80+W?@T9=_WG#ziTFiV_1=h{6Soao7CMYLg$JDLJI=*^RG5@e z*$d@5x^7O>8qHPHTDQ;;P0<_82Trq9Y1FmEgjq1hh<4+QN9%soL>s-|c3iXB_{gj| zu4P-UrTyBsX}BBfjxl9;jn2p!<%(0aTgKFfn+9)*n5x(?`)<`neHCx@cC~7uiy2dB zsZP%5A{tYhTU)tcW!Rttgn&@Y!R$Ba5478-pz~m3t@Vu zrKdwEK=*mU5LYRwV9Q;g)z z{Dhew$u74mReQuSdt%Hk-Wpr6Ehm37?@W#rZF||ep+CkS QM;H8%Z~F;Zz&|yA0a8}QDTc+vM_ADMz*iI(ZJuRnw^ zGDZpwgpnZ3tpwq!+_Bel-uE^5sWeGH|Kk3m!7@aLT}~3R#lg5_4}RU+;9#-~He3`c zOCq{qXs5p*>6+S$A@oQkoe?6YWj?KIvdBbW?z1YA*R@DK1andU9xZ1N!(fQl$P|L@ zJ%*-bt`365*R`;jYXY;7#@$1TsdNr|BzlT1&{gm0AVTN8#~q2zc#nsYZ{3Uf)^3ze zdk*`u@30;7!FCmw0f8J{o=&#b%;ufN*3zn7GiicQc}&uzsznNtmN1)5bA7Q|Uv40a z%4`{N6=Y3zs;xGauz$G5ZO5+J^)~k(Y_*%6rMA034xnffj&j|s*<)lj#X%(Pa@}sV zZEG>SL^*qj;x0is3Fd@{Pz5v7uo|6;V{;OT2 zMzkm_USw1SP)y7cCg~~tVOmA0>IIc3I@g6tOky)^F0Gb>5)Gah8TU&nup*l%eS}R2 zdzzqESXZI1lJo)&o<0WPVn}JBZckOPsce<4rWeX?scWJD!6FLZ9fi^VGEN@eh+TY#qu%9c zpQ_X#=|iA-a*v-T9pA%V^o%_`;-E)j4_$uM zxQY}{0rl!QRKG%{TWmVr1%t?NkZ0a$PUiV3XZvh`=XqaF(L`Ui{zWm^$=r@OI~+eh6`#4oz4J0f~U+%{-g=rH0G?s2Yf zIT!n0VNzOUzbRMHbu*gQXs%? z$8k(_F{6Z*YG;fdqEXt|+|2kZg9hy*_=I8(X0JiN<%WT7X|LE#*q0jhwpp#W=0N4r zY@VAp)U{6%|0g>YQu!)@qS{}l+vnfV$=m3EyssetZrB$JHMlkXWkygG=d8P9H*J99 z-n$hMTaxe&GXgdy{B0-obyR z)*SUZ#b_>@%V%?=j#7X#bDt-HxamSSlrA|g_Xml9-jM%`p~*X+)w z6%lgip`{)?)elbCM{6())gMR$;!%zET*!|@sCXiXMqj2Me*XKjbd0fYSTH6$Oho1`s}e$c)EfbrnMM5eW3HE+zd7M7iwMbm`JLy{&{ZBml7wAE}{ zt8>-*Vgp%JrYeZ5Bx|x;ZFy9}{^8o=xK7Qfd)&XTn%f}Ezoa4s;|jrP3ja1ymT*M&+jE9l^P`X2xx)a!^hM}Jora|JVxy9DcI$7dnylp2mQVi{4sQg_pp;ZVGloX z(C1fJi_q|-hjTF~V)Ny#=qC>_H7wBZR$d=(+xrEx~eNc46 zLLBl=29O(RzNc*k*RVb$<7gZM>7P#l`T@J0fOl>`mG%3m|B3iTSM}eBo)@nqO1zEPNzQ`sNN0J?5o(;Ces(^|365KYk=&09{hRc+L@)Tmi9M~HUuh)3(D zYod+bY&)LWY`kyQ9M7^X&(a?EtQp?wiero!exoz;#$wfR?Uph2-kQN%BBrV~%z>NR zs4wBI-gaFJUCbCmOLg)_2hkW?TVKzIE29P-B7}ru4(6aizw3vAZs{yJP1qM2b89>R%;Fhono+H7Dmm&;P8CA>e_?$e6dg-EsR{TOY@gUi_Svn%IJt)zC5yETTbCp c!6}ZEZF}CjsNcsQM<@J{&w2@&!M`