Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-02-25 20:48:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(avro-json-path-expressions): allow more complex jsonpath expressions (#14149)

Browse Source
This commit is contained in:
robo220 2023-04-26 10:28:11 +01:00 committed by GitHub
parent e4d99c3e26
commit 5db7396c78
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 32 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java
View File

@ -163,4 +163,14 @@ public class GenericAvroJsonProvider extends FlattenerJsonProvider
}
return ImmutableMap.of();
}
@Override
public Object unwrap(final Object o)
{
if (o instanceof Utf8) {
return o.toString();
}
return o;
}
}

8
extensions-core/avro-extensions/src/test/java/org/apache/druid/data/input/AvroStreamInputFormatTest.java
View File

@ -102,10 +102,12 @@ public class AvroStreamInputFormatTest extends InitializedNullHandlingTest
private static final String EVENT_TYPE = "eventType";
private static final String ID = "id";
private static final String SOME_OTHER_ID = "someOtherId";
private static final String NESTED_ARRAY_VAL = "nestedArrayVal";
private static final String IS_VALID = "isValid";
private static final String TOPIC = "aTopic";
static final List<String> DIMENSIONS = Arrays.asList(EVENT_TYPE, ID, SOME_OTHER_ID, IS_VALID);
static final List<String> DIMENSIONS = Arrays.asList(EVENT_TYPE, ID, SOME_OTHER_ID, IS_VALID, NESTED_ARRAY_VAL);
private static final List<String> DIMENSIONS_SCHEMALESS = Arrays.asList(
NESTED_ARRAY_VAL,
SOME_OTHER_ID,
"someIntArray",
"someFloat",
@ -135,7 +137,9 @@ public class AvroStreamInputFormatTest extends InitializedNullHandlingTest
flattenSpec = new JSONPathSpec(
true,
ImmutableList.of(
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested", "someRecord.subLong")
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested", "someRecord.subLong"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nestedArrayVal", "someRecordArray[?(@.nestedString=='string in record')].nestedString")
)
);
for (Module jacksonModule : new AvroExtensionsModule().getJacksonModules()) {

19
extensions-core/avro-extensions/src/test/java/org/apache/druid/data/input/AvroStreamInputRowParserTest.java
View File

@ -76,6 +76,7 @@ public class AvroStreamInputRowParserTest
private static final String ID = "id";
private static final String SOME_OTHER_ID = "someOtherId";
private static final String IS_VALID = "isValid";
private static final String NESTED_ARRAY_VAL = "nestedArrayVal";
private static final String TOPIC = "aTopic";
private static final String EVENT_TYPE_VALUE = "type-a";
private static final long ID_VALUE = 1976491L;
@ -84,8 +85,9 @@ public class AvroStreamInputRowParserTest
private static final int SOME_INT_VALUE = 1;
private static final long SOME_LONG_VALUE = 679865987569912369L;
private static final ZonedDateTime DATE_TIME = ZonedDateTime.of(2015, 10, 25, 19, 30, 0, 0, ZoneOffset.UTC);
static final List<String> DIMENSIONS = Arrays.asList(EVENT_TYPE, ID, SOME_OTHER_ID, IS_VALID);
static final List<String> DIMENSIONS = Arrays.asList(EVENT_TYPE, ID, SOME_OTHER_ID, IS_VALID, NESTED_ARRAY_VAL);
private static final List<String> DIMENSIONS_SCHEMALESS = Arrays.asList(
NESTED_ARRAY_VAL,
SOME_OTHER_ID,
"someIntArray",
"someFloat",
@ -105,7 +107,8 @@ public class AvroStreamInputRowParserTest
new JSONPathSpec(
true,
ImmutableList.of(
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested", "someRecord.subLong")
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested", "someRecord.subLong"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nestedArrayVal", "someRecordArray[?(@.nestedString=='string in record')].nestedString")
)
)
);
@ -115,7 +118,9 @@ public class AvroStreamInputRowParserTest
new JSONPathSpec(
true,
ImmutableList.of(
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested", "someRecord.subLong")
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nested", "someRecord.subLong"),
new JSONPathFieldSpec(JSONPathFieldType.PATH, "nestedArrayVal", "someRecordArray[?(@.nestedString=='string in record')].nestedString")
)
)
);
@ -365,6 +370,14 @@ public class AvroStreamInputRowParserTest
Assert.assertEquals(4892, someRecord.get("subInt"));
Assert.assertEquals(1543698L, someRecord.get("subLong"));
final Object someList = inputRow.getDimension("nestedArrayVal");
Assert.assertNotNull(someList);
Assert.assertTrue(someList instanceof List);
List someRecordObj3List = (List) someList;
Assert.assertEquals(1, someRecordObj3List.size());
Assert.assertEquals("string in record", someRecordObj3List.get(0));
// towards Map avro field as druid dimension, need to convert its toString() back to HashMap to check equality
Assert.assertEquals(1, inputRow.getDimension("someIntValueMap").size());
Assert.assertEquals(