Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1539 from mvfast/cdh_workaround 

Cdh workaround
This commit is contained in:
Fangjin Yang 2015-07-20 09:16:42 -07:00
parent ce295a02cf 7211791585
commit 020827349e
2 changed files with 134 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
docs/content/operations/other-hadoop.md
View File

@ -19,6 +19,32 @@ Members of the community have reported dependency conflicts between the version
For more about building Druid, please see [Building Druid](../development/build.html).
Another workaround solution is to build a custom fat jar of Druid using [sbt](http://www.scala-sbt.org/), which manually excludes all the conflicting Jackson dependencies, and then put this fat jar in the classpath of the command that starts overlord indexing service. To do this, please follow the following steps.
(1) Download and install sbt.
(2) Make a new directory named 'druid_build'.
(3) Cd to 'druid_build' and create the build.sbt file with the content [here](./use_sbt_to_build_fat_jar.md).
You can always add more building targets or remove the ones you don't need.
(4) In the same directory creat a new directory named 'project'.
(5) Put the druid source code into 'druid_build/project'.
(6) Create a file 'druid_build/project/assembly.sbt' with content as follows.
```
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0")
```
(7) In the 'druid_build' directory, run 'sbt assembly'.
(8) In the 'druid_build/target/scala-2.10' folder, you will find the fat jar you just build.
(9) Make sure the jars you've uploaded has been completely removed. The hdfs directory is by default '/tmp/druid-indexing/classpath'.
(10) Include the fat jar in the classpath when you start the indexing service. Make sure you've removed 'lib/*' from your classpath because now the fat jar includes all you need.
Working with Hadoop 1.x and older
---------------------------------

108
docs/content/operations/use_sbt_to_build_fat_jar.md Normal file
View File

@ -0,0 +1,108 @@
---
layout: doc_page
---
Content for build.sbt
---------------------
```scala
libraryDependencies ++= Seq(
"com.amazonaws" % "aws-java-sdk" % "1.9.23" exclude("common-logging", "common-logging"),
"org.joda" % "joda-convert" % "1.7",
"joda-time" % "joda-time" % "2.7",
"io.druid" % "druid" % "0.7.3" excludeAll (
ExclusionRule("org.ow2.asm"),
ExclusionRule("com.fasterxml.jackson.core"),
ExclusionRule("com.fasterxml.jackson.datatype"),
ExclusionRule("com.fasterxml.jackson.dataformat"),
ExclusionRule("com.fasterxml.jackson.jaxrs"),
ExclusionRule("com.fasterxml.jackson.module")
),
"io.druid" % "druid-services" % "0.7.3" excludeAll (
ExclusionRule("org.ow2.asm"),
ExclusionRule("com.fasterxml.jackson.core"),
ExclusionRule("com.fasterxml.jackson.datatype"),
ExclusionRule("com.fasterxml.jackson.dataformat"),
ExclusionRule("com.fasterxml.jackson.jaxrs"),
ExclusionRule("com.fasterxml.jackson.module")
),
"io.druid" % "druid-indexing-service" % "0.7.3" excludeAll (
ExclusionRule("org.ow2.asm"),
ExclusionRule("com.fasterxml.jackson.core"),
ExclusionRule("com.fasterxml.jackson.datatype"),
ExclusionRule("com.fasterxml.jackson.dataformat"),
ExclusionRule("com.fasterxml.jackson.jaxrs"),
ExclusionRule("com.fasterxml.jackson.module")
),
"io.druid" % "druid-indexing-hadoop" % "0.7.3" excludeAll (
ExclusionRule("org.ow2.asm"),
ExclusionRule("com.fasterxml.jackson.core"),
ExclusionRule("com.fasterxml.jackson.datatype"),
ExclusionRule("com.fasterxml.jackson.dataformat"),
ExclusionRule("com.fasterxml.jackson.jaxrs"),
ExclusionRule("com.fasterxml.jackson.module")
),
"io.druid.extensions" % "mysql-metadata-storage" % "0.7.3" excludeAll (
ExclusionRule("org.ow2.asm"),
ExclusionRule("com.fasterxml.jackson.core"),
ExclusionRule("com.fasterxml.jackson.datatype"),
ExclusionRule("com.fasterxml.jackson.dataformat"),
ExclusionRule("com.fasterxml.jackson.jaxrs"),
ExclusionRule("com.fasterxml.jackson.module")
),
"io.druid.extensions" % "druid-s3-extensions" % "0.7.3" excludeAll (
ExclusionRule("org.ow2.asm"),
ExclusionRule("com.fasterxml.jackson.core"),
ExclusionRule("com.fasterxml.jackson.datatype"),
ExclusionRule("com.fasterxml.jackson.dataformat"),
ExclusionRule("com.fasterxml.jackson.jaxrs"),
ExclusionRule("com.fasterxml.jackson.module")
),
"io.druid.extensions" % "druid-histogram" % "0.7.3" excludeAll (
ExclusionRule("org.ow2.asm"),
ExclusionRule("com.fasterxml.jackson.core"),
ExclusionRule("com.fasterxml.jackson.datatype"),
ExclusionRule("com.fasterxml.jackson.dataformat"),
ExclusionRule("com.fasterxml.jackson.jaxrs"),
ExclusionRule("com.fasterxml.jackson.module")
),
"io.druid.extensions" % "druid-hdfs-storage" % "0.7.3" excludeAll (
ExclusionRule("org.ow2.asm"),
ExclusionRule("com.fasterxml.jackson.core"),
ExclusionRule("com.fasterxml.jackson.datatype"),
ExclusionRule("com.fasterxml.jackson.dataformat"),
ExclusionRule("com.fasterxml.jackson.jaxrs"),
ExclusionRule("com.fasterxml.jackson.module")
),
"com.fasterxml.jackson.core" % "jackson-annotations" % "2.3.0",
"com.fasterxml.jackson.core" % "jackson-core" % "2.3.0",
"com.fasterxml.jackson.core" % "jackson-databind" % "2.3.0",
"com.fasterxml.jackson.datatype" % "jackson-datatype-guava" % "2.3.0",
"com.fasterxml.jackson.datatype" % "jackson-datatype-joda" % "2.3.0",
"com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-base" % "2.3.0",
"com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-json-provider" % "2.3.0",
"com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-smile-provider" % "2.3.0",
"com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % "2.3.0",
"com.sun.jersey" % "jersey-servlet" % "1.17.1",
"mysql" % "mysql-connector-java" % "5.1.34",
"org.scalatest" %% "scalatest" % "2.2.3" % "test",
"org.mockito" % "mockito-core" % "1.10.19" % "test"
)
assemblyMergeStrategy in assembly := {
case path if path contains "pom." => MergeStrategy.first
case path if path contains "javax.inject.Named" => MergeStrategy.first
case path if path contains "mime.types" => MergeStrategy.first
case path if path contains "org/apache/commons/logging/impl/SimpleLog.class" => MergeStrategy.first
case path if path contains "org/apache/commons/logging/impl/SimpleLog$1.class" => MergeStrategy.first
case path if path contains "org/apache/commons/logging/impl/NoOpLog.class" => MergeStrategy.first
case path if path contains "org/apache/commons/logging/LogFactory.class" => MergeStrategy.first
case path if path contains "org/apache/commons/logging/LogConfigurationException.class" => MergeStrategy.first
case path if path contains "org/apache/commons/logging/Log.class" => MergeStrategy.first
case path if path contains "META-INF/jersey-module-version" => MergeStrategy.first
case path if path contains ".properties" => MergeStrategy.first
case path if path contains ".class" => MergeStrategy.first
case x =>
val oldStrategy = (assemblyMergeStrategy in assembly).value
oldStrategy(x)
}
```