Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add more tests to ORC format

This commit is contained in:
frank chen 2021-06-18 14:51:00 +08:00
parent 5fa7412c7a
commit fc5f028c97
2 changed files with 57 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
extensions-core/orc-extensions/example/test_json_path_functions.orc Normal file
View File

Binary file not shown.

58
extensions-core/orc-extensions/src/test/java/org/apache/druid/data/input/orc/OrcReaderTest.java
View File

@ -112,7 +112,11 @@ public class OrcReaderTest
ImmutableList.of(
new JSONPathFieldSpec(JSONPathFieldType.PATH, "struct_list_struct_int", "$.middle.list[1].int1"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "struct_list_struct_intlist", "$.middle.list[*].int1"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "struct_list_struct_middleListLength", "$.middle.list.length()"),
new JSONPathFieldSpec(
JSONPathFieldType.PATH,
"struct_list_struct_middleListLength",
"$.middle.list.length()"
),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "list_struct_string", "$.list[0].string1"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "map_struct_int", "$.map.chani.int1")
)
@ -257,6 +261,58 @@ public class OrcReaderTest
}
}
/**
* schema: struct<string1:string, list:array<int>, ts:timestamp>
* data: {"dim1","[7,8,9]","2000-03-12 15:00:00"}
*/
@Test
public void testJsonPathFunctions() throws IOException
{
final OrcInputFormat inputFormat = new OrcInputFormat(
new JSONPathSpec(
true,
ImmutableList.of(
new JSONPathFieldSpec(JSONPathFieldType.PATH, "min", "$.list.min()"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "max", "$.list.max()"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "avg", "$.list.avg()"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "len", "$.list.length()"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "sum", "$.list.sum()"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "stddev", "$.list.stddev()"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "append", "$.list.append(10)")
)
),
null,
new Configuration()
);
final InputEntityReader reader = createReader(
new TimestampSpec("ts", "millis", null),
new DimensionsSpec(null),
inputFormat,
"example/test_json_path_functions.orc"
);
try (CloseableIterator<InputRow> iterator = reader.read()) {
int actualRowCount = 0;
while (iterator.hasNext()) {
final InputRow row = iterator.next();
actualRowCount++;
Assert.assertEquals("7.0", Iterables.getOnlyElement(row.getDimension("min")));
Assert.assertEquals("8.0", Iterables.getOnlyElement(row.getDimension("avg")));
Assert.assertEquals("9.0", Iterables.getOnlyElement(row.getDimension("max")));
Assert.assertEquals("24.0", Iterables.getOnlyElement(row.getDimension("sum")));
Assert.assertEquals("3", Iterables.getOnlyElement(row.getDimension("len")));
//deviation of [7,8,9] is 1/3, stddev is sqrt(1/3), approximately 0.8165
Assert.assertEquals(0.8165, Double.parseDouble(Iterables.getOnlyElement(row.getDimension("stddev"))), 0.0001);
//append is not supported
Assert.assertEquals(Collections.emptyList(), row.getDimension("append"));
}
Assert.assertEquals(1, actualRowCount);
}
}
private InputEntityReader createReader(
TimestampSpec timestampSpec,
DimensionsSpec dimensionsSpec,