From a9a6682a0e8d12fc68a7daa60e0706431c1d4316 Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Wed, 26 Mar 2014 11:43:31 +0530 Subject: [PATCH 1/3] Upgrade to Hadoop 2.3.0 --- hdfs-storage/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- .../io/druid/indexing/common/task/HadoopIndexTask.java | 2 +- pom.xml | 6 +++--- .../io/druid/data/input/ProtoTestEventWrapper.java | 10 +++++++++- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index ebc7b4609fa..4aad536993d 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -52,7 +52,7 @@ org.apache.hadoop - hadoop-core + hadoop-client compile diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index e7ea2010162..16f0efd30f6 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -67,7 +67,7 @@ org.apache.hadoop - hadoop-core + hadoop-client com.fasterxml.jackson.core diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 81b31b48131..ee763f0223c 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -71,7 +71,7 @@ org.apache.hadoop - hadoop-core + hadoop-client test diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java index 878f950f0c4..b4e76814075 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java @@ -66,7 +66,7 @@ public class HadoopIndexTask extends AbstractTask extensionsConfig = Initialization.makeStartupInjector().getInstance(ExtensionsConfig.class); } - private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3"; + private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-client:2.3.0"; @JsonIgnore private final HadoopDruidIndexerSchema schema; @JsonIgnore diff --git a/pom.xml b/pom.xml index 7ec87899fad..225e19f32c1 100644 --- a/pom.xml +++ b/pom.xml @@ -373,7 +373,7 @@ com.google.protobuf protobuf-java - 2.4.0a + 2.5.0 io.tesla.aether @@ -402,8 +402,8 @@ org.apache.hadoop - hadoop-core - 1.0.3 + hadoop-client + 2.3.0 provided diff --git a/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java b/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java index 965859fecfc..e79259554ac 100644 --- a/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java +++ b/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java @@ -23,6 +23,7 @@ package io.druid.data.input; import com.google.protobuf.AbstractMessage; +import com.google.protobuf.UnknownFieldSet; public final class ProtoTestEventWrapper { private ProtoTestEventWrapper() {} @@ -85,7 +86,13 @@ public final class ProtoTestEventWrapper { public ProtoTestEvent getDefaultInstanceForType() { return defaultInstance; } - + + @Override + public UnknownFieldSet getUnknownFields() + { + return UnknownFieldSet.getDefaultInstance(); + } + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { return ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_descriptor; @@ -1049,6 +1056,7 @@ public final class ProtoTestEventWrapper { new com.google.protobuf.Descriptors.FileDescriptor[] { }, assigner); } + // @@protoc_insertion_point(outer_class_scope) } From 06b53f02a8fe929e868636cf80f1446caa2008ae Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Thu, 27 Mar 2014 03:37:45 +0530 Subject: [PATCH 2/3] upgrade default version in CluHadoopIndexer and merge from master --- .../src/main/java/io/druid/cli/CliHadoopIndexer.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java index 0d2ccff8463..357a89771ab 100644 --- a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java +++ b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java @@ -45,18 +45,21 @@ import java.util.List; ) public class CliHadoopIndexer implements Runnable { + + private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-client:2.3.0"; + private static final Logger log = new Logger(CliHadoopIndexer.class); @Arguments(description = "A JSON object or the path to a file that contains a JSON object", required = true) private String argumentSpec; @Option(name = "hadoop", - description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-core:1.0.3") - private String hadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3"; + description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-client:2.3.0") + private String hadoopCoordinates = defaultHadoopCoordinates; @Option(name = "hadoopDependencies", - description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-core:1.0.3") - private List hadoopDependencyCoordinates = Arrays.asList("org.apache.hadoop:hadoop-core:1.0.3"); + description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-client:2.3.0") + private List hadoopDependencyCoordinates = Arrays.asList(defaultHadoopCoordinates); @Inject private ExtensionsConfig extensionsConfig = null; From 298b9c9346eb6a8bb680ace4163955369c091c7d Mon Sep 17 00:00:00 2001 From: nishantmonu51 Date: Thu, 27 Mar 2014 20:16:49 +0530 Subject: [PATCH 3/3] update default version in docs --- docs/content/Tasks.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/Tasks.md b/docs/content/Tasks.md index d2b90d6fcb0..a1fa7beeeb2 100644 --- a/docs/content/Tasks.md +++ b/docs/content/Tasks.md @@ -74,14 +74,14 @@ The Hadoop Index Task is used to index larger data sets that require the paralle |--------|-----------|---------| |type|The task type, this should always be "index_hadoop".|yes| |config|A Hadoop Index Config. See [Batch Ingestion](Batch-ingestion.html)|yes| -|hadoopCoordinates|The Maven \:\:\ of Hadoop to use. The default is "org.apache.hadoop:hadoop-core:1.0.3".|no| +|hadoopCoordinates|The Maven \:\:\ of Hadoop to use. The default is "org.apache.hadoop:hadoop-client:2.3.0".|no| The Hadoop Index Config submitted as part of an Hadoop Index Task is identical to the Hadoop Index Config used by the `HadoopBatchIndexer` except that three fields must be omitted: `segmentOutputPath`, `workingPath`, `updaterJobSpec`. The Indexing Service takes care of setting these fields internally. #### Using your own Hadoop distribution -Druid is compiled against Apache hadoop-core 1.0.3. However, if you happen to use a different flavor of hadoop that is API compatible with hadoop-core 1.0.3, you should only have to change the hadoopCoordinates property to point to the maven artifact used by your distribution. +Druid is compiled against Apache hadoop-client 2.3.0. However, if you happen to use a different flavor of hadoop that is API compatible with hadoop-client 2.3.0, you should only have to change the hadoopCoordinates property to point to the maven artifact used by your distribution. #### Resolving dependency conflicts running HadoopIndexTask