From c2add5730bd173f99a163ecda9feca3a9273825d Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 2 Dec 2014 11:01:12 -0800 Subject: [PATCH] Fix Hadoop CLI jobs * Change "schema" --> "spec" for cli hadoop to keep up with internal hadoop * Added check for HadoopDruidIndexerConfig deserialization from Map to see if it is trying to get a HadoopDruidIndexerConfig or a HadoopIngestionSpec --- .../indexer/HadoopDruidIndexerConfig.java | 28 +++++++++++++------ .../java/io/druid/indexer/HadoopIOConfig.java | 2 ++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java index e33a9c30fb5..e54803cee72 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopDruidIndexerConfig.java @@ -114,12 +114,21 @@ public class HadoopDruidIndexerConfig public static HadoopDruidIndexerConfig fromMap(Map argSpec) { - return new HadoopDruidIndexerConfig( - HadoopDruidIndexerConfig.jsonMapper.convertValue( - argSpec, - HadoopIngestionSpec.class - ) + // Eventually PathSpec needs to get rid of its Hadoop dependency, then maybe this can be ingested directly without + // the Map<> intermediary + + if(argSpec.containsKey("spec")){ + return HadoopDruidIndexerConfig.jsonMapper.convertValue( + argSpec, + HadoopDruidIndexerConfig.class ); + } + return new HadoopDruidIndexerConfig( + HadoopDruidIndexerConfig.jsonMapper.convertValue( + argSpec, + HadoopIngestionSpec.class + ) + ); } @SuppressWarnings("unchecked") @@ -142,12 +151,13 @@ public class HadoopDruidIndexerConfig @SuppressWarnings("unchecked") public static HadoopDruidIndexerConfig fromString(String str) { + // This is a map to try and prevent dependency screwbally-ness try { return fromMap( (Map) HadoopDruidIndexerConfig.jsonMapper.readValue( str, new TypeReference>() - { - } + { + } ) ); } @@ -171,7 +181,7 @@ public class HadoopDruidIndexerConfig @JsonCreator public HadoopDruidIndexerConfig( - final @JsonProperty("schema") HadoopIngestionSpec schema + final @JsonProperty("spec") HadoopIngestionSpec schema ) { this.schema = schema; @@ -202,7 +212,7 @@ public class HadoopDruidIndexerConfig this.rollupGran = schema.getDataSchema().getGranularitySpec().getQueryGranularity(); } - @JsonProperty + @JsonProperty(value="spec") public HadoopIngestionSpec getSchema() { return schema; diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIOConfig.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIOConfig.java index 3aca5b6136b..fdd858e2b27 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIOConfig.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIOConfig.java @@ -19,6 +19,7 @@ package io.druid.indexer; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import io.druid.indexer.updater.MetadataStorageUpdaterJobSpec; @@ -35,6 +36,7 @@ public class HadoopIOConfig implements IOConfig private final MetadataStorageUpdaterJobSpec metadataUpdateSpec; private final String segmentOutputPath; + @JsonCreator public HadoopIOConfig( final @JsonProperty("inputSpec") Map pathSpec, final @JsonProperty("metadataUpdateSpec") MetadataStorageUpdaterJobSpec metadataUpdateSpec,