Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #445 from metamx/hadoop-version-update 

Hadoop version update
This commit is contained in:
fjy 2014-04-17 19:16:44 -06:00
parent a520a2dde0 298b9c9346
commit c00fb1d08e
8 changed files with 25 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/content/Tasks.md
View File

@ -74,14 +74,14 @@ The Hadoop Index Task is used to index larger data sets that require the paralle
|--------|-----------|---------|
|type|The task type, this should always be "index_hadoop".|yes|
|config|A Hadoop Index Config. See [Batch Ingestion](Batch-ingestion.html)|yes|
|hadoopCoordinates|The Maven \<groupId\>:\<artifactId\>:\<version\> of Hadoop to use. The default is "org.apache.hadoop:hadoop-core:1.0.3".|no|
|hadoopCoordinates|The Maven \<groupId\>:\<artifactId\>:\<version\> of Hadoop to use. The default is "org.apache.hadoop:hadoop-client:2.3.0".|no|
The Hadoop Index Config submitted as part of an Hadoop Index Task is identical to the Hadoop Index Config used by the `HadoopBatchIndexer` except that three fields must be omitted: `segmentOutputPath`, `workingPath`, `updaterJobSpec`. The Indexing Service takes care of setting these fields internally.
#### Using your own Hadoop distribution
Druid is compiled against Apache hadoop-core 1.0.3. However, if you happen to use a different flavor of hadoop that is API compatible with hadoop-core 1.0.3, you should only have to change the hadoopCoordinates property to point to the maven artifact used by your distribution.
Druid is compiled against Apache hadoop-client 2.3.0. However, if you happen to use a different flavor of hadoop that is API compatible with hadoop-client 2.3.0, you should only have to change the hadoopCoordinates property to point to the maven artifact used by your distribution.
#### Resolving dependency conflicts running HadoopIndexTask

2
hdfs-storage/pom.xml
View File

@ -52,7 +52,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<artifactId>hadoop-client</artifactId>
<scope>compile</scope>
</dependency>
<dependency>

2
indexing-hadoop/pom.xml
View File

@ -67,7 +67,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>

2
indexing-service/pom.xml
View File

@ -71,7 +71,7 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<artifactId>hadoop-client</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

2
indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java
View File

@ -66,7 +66,7 @@ public class HadoopIndexTask extends AbstractTask
extensionsConfig = Initialization.makeStartupInjector().getInstance(ExtensionsConfig.class);
}
private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3";
private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-client:2.3.0";
@JsonIgnore
private final HadoopDruidIndexerSchema schema;
@JsonIgnore

6
pom.xml
View File

@ -373,7 +373,7 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.4.0a</version>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>io.tesla.aether</groupId>
@ -402,8 +402,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>1.0.3</version>
<artifactId>hadoop-client</artifactId>
<version>2.3.0</version>
<scope>provided</scope>
</dependency>

10
processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java
View File

@ -23,6 +23,7 @@
package io.druid.data.input;
import com.google.protobuf.AbstractMessage;
import com.google.protobuf.UnknownFieldSet;
public final class ProtoTestEventWrapper {
private ProtoTestEventWrapper() {}
@ -85,7 +86,13 @@ public final class ProtoTestEventWrapper {
public ProtoTestEvent getDefaultInstanceForType() {
return defaultInstance;
}
@Override
public UnknownFieldSet getUnknownFields()
{
return UnknownFieldSet.getDefaultInstance();
}
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_descriptor;
@ -1049,6 +1056,7 @@ public final class ProtoTestEventWrapper {
new com.google.protobuf.Descriptors.FileDescriptor[] {
}, assigner);
}
// @@protoc_insertion_point(outer_class_scope)
}

11
services/src/main/java/io/druid/cli/CliHadoopIndexer.java
View File

@ -45,18 +45,21 @@ import java.util.List;
)
public class CliHadoopIndexer implements Runnable
{
private static String defaultHadoopCoordinates = "org.apache.hadoop:hadoop-client:2.3.0";
private static final Logger log = new Logger(CliHadoopIndexer.class);
@Arguments(description = "A JSON object or the path to a file that contains a JSON object", required = true)
private String argumentSpec;
@Option(name = "hadoop",
description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-core:1.0.3")
private String hadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3";
description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-client:2.3.0")
private String hadoopCoordinates = defaultHadoopCoordinates;
@Option(name = "hadoopDependencies",
description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-core:1.0.3")
private List<String> hadoopDependencyCoordinates = Arrays.<String>asList("org.apache.hadoop:hadoop-core:1.0.3");
description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-client:2.3.0")
private List<String> hadoopDependencyCoordinates = Arrays.<String>asList(defaultHadoopCoordinates);
@Inject
private ExtensionsConfig extensionsConfig = null;