From dacee9ed202a401fd73f0edb8148ace4a8614cfd Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 14 Aug 2013 16:51:02 -0700 Subject: [PATCH 1/3] Fix various InputRowParser serde issues --- .../druid/indexer/data/InputRowParser.java | 7 -- .../indexer/data/StringInputRowParser.java | 44 ++++++++---- .../indexer/data/InputRowParserSerdeTest.java | 70 +++++++++++++++++++ .../firehose/KafkaFirehoseFactory.java | 29 ++++---- 4 files changed, 112 insertions(+), 38 deletions(-) create mode 100644 indexing-common/src/test/java/com/metamx/druid/indexer/data/InputRowParserSerdeTest.java diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java b/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java index 960ac8719a7..231cbd44102 100644 --- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java +++ b/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java @@ -1,15 +1,8 @@ package com.metamx.druid.indexer.data; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.metamx.common.exception.FormattedException; import com.metamx.druid.input.InputRow; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = StringInputRowParser.class) -@JsonSubTypes({ - @JsonSubTypes.Type(name = "string", value = StringInputRowParser.class), - @JsonSubTypes.Type(name = "protobuf", value = ProtoBufInputRowParser.class) -}) public interface InputRowParser { public InputRow parse(T input) throws FormattedException; diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/StringInputRowParser.java b/indexing-common/src/main/java/com/metamx/druid/indexer/data/StringInputRowParser.java index c1b52d142c6..3d23d5d1dde 100644 --- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/StringInputRowParser.java +++ b/indexing-common/src/main/java/com/metamx/druid/indexer/data/StringInputRowParser.java @@ -19,6 +19,15 @@ package com.metamx.druid.indexer.data; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; +import com.metamx.common.exception.FormattedException; +import com.metamx.common.parsers.Parser; +import com.metamx.common.parsers.ToLowerCaseParser; +import com.metamx.druid.input.InputRow; + import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.CoderResult; @@ -26,21 +35,13 @@ import java.nio.charset.CodingErrorAction; import java.util.List; import java.util.Map; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonValue; -import com.google.common.base.Charsets; -import com.metamx.common.exception.FormattedException; -import com.metamx.common.parsers.Parser; -import com.metamx.common.parsers.ToLowerCaseParser; -import com.metamx.druid.input.InputRow; - /** */ public class StringInputRowParser implements ByteBufferInputRowParser { - private final InputRowParser> inputRowCreator; + private final MapInputRowParser inputRowCreator; private final Parser parser; + private final DataSpec dataSpec; private CharBuffer chars = null; @@ -50,6 +51,7 @@ public class StringInputRowParser implements ByteBufferInputRowParser @JsonProperty("data") DataSpec dataSpec, @JsonProperty("dimensionExclusions") List dimensionExclusions) { + this.dataSpec = dataSpec; this.inputRowCreator = new MapInputRowParser(timestampSpec, dataSpec.getDimensions(), dimensionExclusions); this.parser = new ToLowerCaseParser(dataSpec.getParser()); } @@ -116,9 +118,21 @@ public class StringInputRowParser implements ByteBufferInputRowParser return inputRowCreator.parse(theMap); } - @JsonValue - public InputRowParser> getInputRowCreator() - { - return inputRowCreator; - } + @JsonProperty + public TimestampSpec getTimestampSpec() + { + return inputRowCreator.getTimestampSpec(); + } + + @JsonProperty("data") + public DataSpec getDataSpec() + { + return dataSpec; + } + + @JsonProperty + public List getDimensionExclusions() + { + return ImmutableList.copyOf(inputRowCreator.getDimensionExclusions()); + } } diff --git a/indexing-common/src/test/java/com/metamx/druid/indexer/data/InputRowParserSerdeTest.java b/indexing-common/src/test/java/com/metamx/druid/indexer/data/InputRowParserSerdeTest.java new file mode 100644 index 00000000000..418661ce9d6 --- /dev/null +++ b/indexing-common/src/test/java/com/metamx/druid/indexer/data/InputRowParserSerdeTest.java @@ -0,0 +1,70 @@ +package com.metamx.druid.indexer.data; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.metamx.druid.index.v1.SpatialDimensionSchema; +import com.metamx.druid.input.InputRow; +import com.metamx.druid.jackson.DefaultObjectMapper; +import junit.framework.Assert; +import org.joda.time.DateTime; +import org.junit.Test; + +import java.nio.ByteBuffer; + +public class InputRowParserSerdeTest +{ + private final ObjectMapper jsonMapper = new DefaultObjectMapper(); + + @Test + public void testStringInputRowParserSerde() throws Exception + { + final StringInputRowParser parser = new StringInputRowParser( + new TimestampSpec("timestamp", "iso"), + new JSONDataSpec( + ImmutableList.of("foo", "bar"), ImmutableList.of() + ), + ImmutableList.of("baz") + ); + final ByteBufferInputRowParser parser2 = jsonMapper.readValue( + jsonMapper.writeValueAsBytes(parser), + ByteBufferInputRowParser.class + ); + final InputRow parsed = parser2.parse( + ByteBuffer.wrap( + "{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"2000\"}".getBytes(Charsets.UTF_8) + ) + ); + Assert.assertEquals(ImmutableList.of("foo", "bar"), parsed.getDimensions()); + Assert.assertEquals(ImmutableList.of("x"), parsed.getDimension("foo")); + Assert.assertEquals(ImmutableList.of("y"), parsed.getDimension("bar")); + Assert.assertEquals(new DateTime("2000").getMillis(), parsed.getTimestampFromEpoch()); + } + + @Test + public void testMapInputRowParserSerde() throws Exception + { + final MapInputRowParser parser = new MapInputRowParser( + new TimestampSpec("timestamp", "iso"), + ImmutableList.of("foo", "bar"), + ImmutableList.of("baz") + ); + final MapInputRowParser parser2 = jsonMapper.readValue( + jsonMapper.writeValueAsBytes(parser), + MapInputRowParser.class + ); + final InputRow parsed = parser2.parse( + ImmutableMap.of( + "foo", "x", + "bar", "y", + "qux", "z", + "timestamp", "2000" + ) + ); + Assert.assertEquals(ImmutableList.of("foo", "bar"), parsed.getDimensions()); + Assert.assertEquals(ImmutableList.of("x"), parsed.getDimension("foo")); + Assert.assertEquals(ImmutableList.of("y"), parsed.getDimension("bar")); + Assert.assertEquals(new DateTime("2000").getMillis(), parsed.getTimestampFromEpoch()); + } +} diff --git a/realtime/src/main/java/com/metamx/druid/realtime/firehose/KafkaFirehoseFactory.java b/realtime/src/main/java/com/metamx/druid/realtime/firehose/KafkaFirehoseFactory.java index 01debf2a0a2..6edcd3ac4a8 100644 --- a/realtime/src/main/java/com/metamx/druid/realtime/firehose/KafkaFirehoseFactory.java +++ b/realtime/src/main/java/com/metamx/druid/realtime/firehose/KafkaFirehoseFactory.java @@ -19,14 +19,13 @@ package com.metamx.druid.realtime.firehose; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; - +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; +import com.metamx.common.exception.FormattedException; +import com.metamx.common.logger.Logger; import com.metamx.druid.indexer.data.ByteBufferInputRowParser; +import com.metamx.druid.input.InputRow; import kafka.consumer.Consumer; import kafka.consumer.ConsumerConfig; import kafka.consumer.KafkaStream; @@ -34,13 +33,11 @@ import kafka.javaapi.consumer.ConsumerConnector; import kafka.message.Message; import kafka.message.MessageAndMetadata; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableMap; -import com.metamx.common.exception.FormattedException; -import com.metamx.common.logger.Logger; -import com.metamx.druid.indexer.data.InputRowParser; -import com.metamx.druid.input.InputRow; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; /** */ @@ -92,9 +89,9 @@ public class KafkaFirehoseFactory implements FirehoseFactory { private final ConsumerConnector connector; private final Iterator> iter; - private final InputRowParser parser; + private final ByteBufferInputRowParser parser; - public DefaultFirehose(ConsumerConnector connector, KafkaStream stream, InputRowParser parser) + public DefaultFirehose(ConsumerConnector connector, KafkaStream stream, ByteBufferInputRowParser parser) { iter = stream.iterator(); this.connector = connector; From b6c445c726ea1768f337cb55e09ec6a89f52f187 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 14 Aug 2013 17:03:28 -0700 Subject: [PATCH 2/3] [maven-release-plugin] prepare release druid-0.5.40 --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- indexing-common/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index e42370b8555..3ed27ed1185 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 diff --git a/common/pom.xml b/common/pom.xml index 4765d858146..4eb5d19de77 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 diff --git a/examples/pom.xml b/examples/pom.xml index 018572002d7..181a77e51f2 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 diff --git a/indexing-common/pom.xml b/indexing-common/pom.xml index 93ca6cfe3db..7f2e1ea98c3 100644 --- a/indexing-common/pom.xml +++ b/indexing-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 1594c9235aa..63a657ecd65 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index e8a818a7701..652477b82de 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 diff --git a/pom.xml b/pom.xml index 9e61b78e864..7e542823056 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.5.40-SNAPSHOT + 0.5.40 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 984b6f61b93..687b4e2b1cf 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 diff --git a/server/pom.xml b/server/pom.xml index 698adba997b..329af9d45f0 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 diff --git a/services/pom.xml b/services/pom.xml index 4b15e817ce3..82ffcd19dcc 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.5.40-SNAPSHOT + 0.5.40 com.metamx druid - 0.5.40-SNAPSHOT + 0.5.40 From a0fcd02b3cd6297864344783ec935d68b39a7b38 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 14 Aug 2013 17:03:40 -0700 Subject: [PATCH 3/3] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- examples/pom.xml | 2 +- indexing-common/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 3ed27ed1185..79974cb5819 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 4eb5d19de77..e42e0b0980c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 181a77e51f2..175e1512e38 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -9,7 +9,7 @@ com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT diff --git a/indexing-common/pom.xml b/indexing-common/pom.xml index 7f2e1ea98c3..1f2bd49069e 100644 --- a/indexing-common/pom.xml +++ b/indexing-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 63a657ecd65..0025db8fcc1 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 652477b82de..93250acbe0f 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT diff --git a/pom.xml b/pom.xml index 7e542823056..1f8f6227a91 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.5.40 + 0.5.41-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 687b4e2b1cf..6382a95cc97 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 329af9d45f0..338454de5d0 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 82ffcd19dcc..9ed93dc23ec 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.5.40 + 0.5.41-SNAPSHOT com.metamx druid - 0.5.40 + 0.5.41-SNAPSHOT