Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix avro parser issue while deserializing sketches (#6440) 

* Fix object transform for avro parser

* Remove unwanted space
This commit is contained in:
Atul Mohan 2018-10-10 10:53:26 -05:00 committed by Charles Allen
parent 1fa045862a
commit e69a2f217b
3 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/AvroFlattenerMaker.java
View File

@ -31,7 +31,6 @@ import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.parsers.ObjectFlatteners; import org.apache.druid.java.util.common.parsers.ObjectFlatteners;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -105,7 +104,7 @@ public class AvroFlattenerMaker implements ObjectFlatteners.FlattenerMaker<Gener
if (binaryAsString) { if (binaryAsString) {
return StringUtils.fromUtf8(((ByteBuffer) field).array()); return StringUtils.fromUtf8(((ByteBuffer) field).array());
} else { } else {
return Arrays.toString(((ByteBuffer) field).array()); return ((ByteBuffer) field).array();
} }
} }
if (field instanceof Utf8) { if (field instanceof Utf8) {

2
extensions-core/avro-extensions/src/test/java/org/apache/druid/data/input/AvroHadoopInputRowParserTest.java
View File

@ -83,7 +83,7 @@ public class AvroHadoopInputRowParserTest
AvroHadoopInputRowParser.class AvroHadoopInputRowParser.class
); );
InputRow inputRow = parser2.parseBatch(record).get(0); InputRow inputRow = parser2.parseBatch(record).get(0);
assertInputRowCorrect(inputRow, DIMENSIONS); assertInputRowCorrect(inputRow, DIMENSIONS, fromPigAvroStorage);
} }

14
extensions-core/avro-extensions/src/test/java/org/apache/druid/data/input/AvroStreamInputRowParserTest.java
View File

@ -216,7 +216,7 @@ public class AvroStreamInputRowParserTest
InputRow inputRow = parser2.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); InputRow inputRow = parser2.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0);
assertInputRowCorrect(inputRow, DIMENSIONS); assertInputRowCorrect(inputRow, DIMENSIONS, false);
} }
@Test @Test
@ -257,10 +257,10 @@ public class AvroStreamInputRowParserTest
InputRow inputRow = parser2.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); InputRow inputRow = parser2.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0);
assertInputRowCorrect(inputRow, DIMENSIONS_SCHEMALESS); assertInputRowCorrect(inputRow, DIMENSIONS_SCHEMALESS, false);
} }
public static void assertInputRowCorrect(InputRow inputRow, List<String> expectedDimensions) public static void assertInputRowCorrect(InputRow inputRow, List<String> expectedDimensions, boolean isFromPigAvro)
{ {
assertEquals(expectedDimensions, inputRow.getDimensions()); assertEquals(expectedDimensions, inputRow.getDimensions());
assertEquals(1543698L, inputRow.getTimestampFromEpoch()); assertEquals(1543698L, inputRow.getTimestampFromEpoch());
@ -307,10 +307,12 @@ public class AvroStreamInputRowParserTest
); );
assertEquals(Collections.singletonList(SOME_UNION_VALUE), inputRow.getDimension("someUnion")); assertEquals(Collections.singletonList(SOME_UNION_VALUE), inputRow.getDimension("someUnion"));
assertEquals(Collections.emptyList(), inputRow.getDimension("someNull")); assertEquals(Collections.emptyList(), inputRow.getDimension("someNull"));
assertEquals(Collections.singletonList(String.valueOf(SOME_FIXED_VALUE)), inputRow.getDimension("someFixed")); if (isFromPigAvro) {
assertEquals(String.valueOf(SOME_FIXED_VALUE), Arrays.toString((byte[]) inputRow.getRaw("someFixed")));
}
assertEquals( assertEquals(
Collections.singletonList(Arrays.toString(SOME_BYTES_VALUE.array())), Arrays.toString(SOME_BYTES_VALUE.array()),
inputRow.getDimension("someBytes") Arrays.toString((byte[]) (inputRow.getRaw("someBytes")))
); );
assertEquals(Collections.singletonList(String.valueOf(MyEnum.ENUM1)), inputRow.getDimension("someEnum")); assertEquals(Collections.singletonList(String.valueOf(MyEnum.ENUM1)), inputRow.getDimension("someEnum"));
assertEquals(Collections.singletonList(String.valueOf(SOME_RECORD_VALUE)), inputRow.getDimension("someRecord")); assertEquals(Collections.singletonList(String.valueOf(SOME_RECORD_VALUE)), inputRow.getDimension("someRecord"));