diff --git a/docs/content/Tasks.md b/docs/content/Tasks.md index 061f02d872a..80f5d0952f6 100644 --- a/docs/content/Tasks.md +++ b/docs/content/Tasks.md @@ -74,14 +74,14 @@ The Hadoop Index Task is used to index larger data sets that require the paralle |--------|-----------|---------| |type|The task type, this should always be "index_hadoop".|yes| |config|A Hadoop Index Config. See [Batch Ingestion](Batch-ingestion.html)|yes| -|hadoopCoordinates|The Maven \:\:\ of Hadoop to use. The default is "org.apache.hadoop:hadoop-core:1.0.3".|no| +|hadoopCoordinates|The Maven \:\:\ of Hadoop to use. The default is "org.apache.hadoop:hadoop-client:2.3.0".|no| The Hadoop Index Config submitted as part of an Hadoop Index Task is identical to the Hadoop Index Config used by the `HadoopBatchIndexer` except that three fields must be omitted: `segmentOutputPath`, `workingPath`, `updaterJobSpec`. The Indexing Service takes care of setting these fields internally. #### Using your own Hadoop distribution -Druid is compiled against Apache hadoop-core 1.0.3. However, if you happen to use a different flavor of hadoop that is API compatible with hadoop-core 1.0.3, you should only have to change the hadoopCoordinates property to point to the maven artifact used by your distribution. +Druid is compiled against Apache hadoop-client 2.3.0. However, if you happen to use a different flavor of hadoop that is API compatible with hadoop-client 2.3.0, you should only have to change the hadoopCoordinates property to point to the maven artifact used by your distribution. #### Resolving dependency conflicts running HadoopIndexTask diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index f3dd0435918..edf370355c5 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -52,7 +52,7 @@ org.apache.hadoop - hadoop-core + hadoop-client compile diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index d3036493ce7..58e1b2c8d44 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -67,7 +67,7 @@ org.apache.hadoop - hadoop-core + hadoop-client com.fasterxml.jackson.core diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 29dbcb5b783..ecbe4a96667 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -71,7 +71,7 @@ org.apache.hadoop - hadoop-core + hadoop-client test diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java index ca38f90e3cc..7e9d4032e21 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java @@ -66,7 +66,7 @@ public class HadoopIndexTask extends AbstractTask extensionsConfig = Initialization.makeStartupInjector().getInstance(ExtensionsConfig.class); } - private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3"; + private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-client:2.3.0"; @JsonIgnore private final HadoopDruidIndexerSchema schema; @JsonIgnore diff --git a/pom.xml b/pom.xml index ca808db16d8..464fd9dfbab 100644 --- a/pom.xml +++ b/pom.xml @@ -373,7 +373,7 @@ com.google.protobuf protobuf-java - 2.4.0a + 2.5.0 io.tesla.aether @@ -402,8 +402,8 @@ org.apache.hadoop - hadoop-core - 1.0.3 + hadoop-client + 2.3.0 provided diff --git a/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java b/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java index 965859fecfc..e79259554ac 100644 --- a/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java +++ b/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java @@ -23,6 +23,7 @@ package io.druid.data.input; import com.google.protobuf.AbstractMessage; +import com.google.protobuf.UnknownFieldSet; public final class ProtoTestEventWrapper { private ProtoTestEventWrapper() {} @@ -85,7 +86,13 @@ public final class ProtoTestEventWrapper { public ProtoTestEvent getDefaultInstanceForType() { return defaultInstance; } - + + @Override + public UnknownFieldSet getUnknownFields() + { + return UnknownFieldSet.getDefaultInstance(); + } + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { return ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_descriptor; @@ -1049,6 +1056,7 @@ public final class ProtoTestEventWrapper { new com.google.protobuf.Descriptors.FileDescriptor[] { }, assigner); } + // @@protoc_insertion_point(outer_class_scope) } diff --git a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java index 3b20a3a235c..cf81cc7aca5 100644 --- a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java +++ b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java @@ -45,18 +45,21 @@ import java.util.List; ) public class CliHadoopIndexer implements Runnable { + + private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-client:2.3.0"; + private static final Logger log = new Logger(CliHadoopIndexer.class); @Arguments(description = "A JSON object or the path to a file that contains a JSON object", required = true) private String argumentSpec; @Option(name = "hadoop", - description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-core:1.0.3") - private String hadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3"; + description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-client:2.3.0") + private String hadoopCoordinates = defaultHadoopCoordinates; @Option(name = "hadoopDependencies", - description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-core:1.0.3") - private List hadoopDependencyCoordinates = Arrays.asList("org.apache.hadoop:hadoop-core:1.0.3"); + description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-client:2.3.0") + private List hadoopDependencyCoordinates = Arrays.asList(defaultHadoopCoordinates); @Inject private ExtensionsConfig extensionsConfig = null;