diff --git a/docs/content/operations/other-hadoop.md b/docs/content/operations/other-hadoop.md index dd58e6f4846..1026a54ddb2 100644 --- a/docs/content/operations/other-hadoop.md +++ b/docs/content/operations/other-hadoop.md @@ -19,6 +19,32 @@ Members of the community have reported dependency conflicts between the version For more about building Druid, please see [Building Druid](../development/build.html). +Another workaround solution is to build a custom fat jar of Druid using [sbt](http://www.scala-sbt.org/), which manually excludes all the conflicting Jackson dependencies, and then put this fat jar in the classpath of the command that starts overlord indexing service. To do this, please follow the following steps. + +(1) Download and install sbt. + +(2) Make a new directory named 'druid_build'. + +(3) Cd to 'druid_build' and create the build.sbt file with the content [here](./use_sbt_to_build_fat_jar.md). + +You can always add more building targets or remove the ones you don't need. + +(4) In the same directory creat a new directory named 'project'. + +(5) Put the druid source code into 'druid_build/project'. + +(6) Create a file 'druid_build/project/assembly.sbt' with content as follows. +``` +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0") +``` + +(7) In the 'druid_build' directory, run 'sbt assembly'. + +(8) In the 'druid_build/target/scala-2.10' folder, you will find the fat jar you just build. + +(9) Make sure the jars you've uploaded has been completely removed. The hdfs directory is by default '/tmp/druid-indexing/classpath'. + +(10) Include the fat jar in the classpath when you start the indexing service. Make sure you've removed 'lib/*' from your classpath because now the fat jar includes all you need. Working with Hadoop 1.x and older --------------------------------- diff --git a/docs/content/operations/use_sbt_to_build_fat_jar.md b/docs/content/operations/use_sbt_to_build_fat_jar.md new file mode 100644 index 00000000000..3d6d88fd421 --- /dev/null +++ b/docs/content/operations/use_sbt_to_build_fat_jar.md @@ -0,0 +1,108 @@ +--- +layout: doc_page +--- + +Content for build.sbt +--------------------- +```scala +libraryDependencies ++= Seq( + "com.amazonaws" % "aws-java-sdk" % "1.9.23" exclude("common-logging", "common-logging"), + "org.joda" % "joda-convert" % "1.7", + "joda-time" % "joda-time" % "2.7", + "io.druid" % "druid" % "0.7.3" excludeAll ( + ExclusionRule("org.ow2.asm"), + ExclusionRule("com.fasterxml.jackson.core"), + ExclusionRule("com.fasterxml.jackson.datatype"), + ExclusionRule("com.fasterxml.jackson.dataformat"), + ExclusionRule("com.fasterxml.jackson.jaxrs"), + ExclusionRule("com.fasterxml.jackson.module") + ), + "io.druid" % "druid-services" % "0.7.3" excludeAll ( + ExclusionRule("org.ow2.asm"), + ExclusionRule("com.fasterxml.jackson.core"), + ExclusionRule("com.fasterxml.jackson.datatype"), + ExclusionRule("com.fasterxml.jackson.dataformat"), + ExclusionRule("com.fasterxml.jackson.jaxrs"), + ExclusionRule("com.fasterxml.jackson.module") + ), + "io.druid" % "druid-indexing-service" % "0.7.3" excludeAll ( + ExclusionRule("org.ow2.asm"), + ExclusionRule("com.fasterxml.jackson.core"), + ExclusionRule("com.fasterxml.jackson.datatype"), + ExclusionRule("com.fasterxml.jackson.dataformat"), + ExclusionRule("com.fasterxml.jackson.jaxrs"), + ExclusionRule("com.fasterxml.jackson.module") + ), + "io.druid" % "druid-indexing-hadoop" % "0.7.3" excludeAll ( + ExclusionRule("org.ow2.asm"), + ExclusionRule("com.fasterxml.jackson.core"), + ExclusionRule("com.fasterxml.jackson.datatype"), + ExclusionRule("com.fasterxml.jackson.dataformat"), + ExclusionRule("com.fasterxml.jackson.jaxrs"), + ExclusionRule("com.fasterxml.jackson.module") + ), + "io.druid.extensions" % "mysql-metadata-storage" % "0.7.3" excludeAll ( + ExclusionRule("org.ow2.asm"), + ExclusionRule("com.fasterxml.jackson.core"), + ExclusionRule("com.fasterxml.jackson.datatype"), + ExclusionRule("com.fasterxml.jackson.dataformat"), + ExclusionRule("com.fasterxml.jackson.jaxrs"), + ExclusionRule("com.fasterxml.jackson.module") + ), + "io.druid.extensions" % "druid-s3-extensions" % "0.7.3" excludeAll ( + ExclusionRule("org.ow2.asm"), + ExclusionRule("com.fasterxml.jackson.core"), + ExclusionRule("com.fasterxml.jackson.datatype"), + ExclusionRule("com.fasterxml.jackson.dataformat"), + ExclusionRule("com.fasterxml.jackson.jaxrs"), + ExclusionRule("com.fasterxml.jackson.module") + ), + "io.druid.extensions" % "druid-histogram" % "0.7.3" excludeAll ( + ExclusionRule("org.ow2.asm"), + ExclusionRule("com.fasterxml.jackson.core"), + ExclusionRule("com.fasterxml.jackson.datatype"), + ExclusionRule("com.fasterxml.jackson.dataformat"), + ExclusionRule("com.fasterxml.jackson.jaxrs"), + ExclusionRule("com.fasterxml.jackson.module") + ), + "io.druid.extensions" % "druid-hdfs-storage" % "0.7.3" excludeAll ( + ExclusionRule("org.ow2.asm"), + ExclusionRule("com.fasterxml.jackson.core"), + ExclusionRule("com.fasterxml.jackson.datatype"), + ExclusionRule("com.fasterxml.jackson.dataformat"), + ExclusionRule("com.fasterxml.jackson.jaxrs"), + ExclusionRule("com.fasterxml.jackson.module") + ), + "com.fasterxml.jackson.core" % "jackson-annotations" % "2.3.0", + "com.fasterxml.jackson.core" % "jackson-core" % "2.3.0", + "com.fasterxml.jackson.core" % "jackson-databind" % "2.3.0", + "com.fasterxml.jackson.datatype" % "jackson-datatype-guava" % "2.3.0", + "com.fasterxml.jackson.datatype" % "jackson-datatype-joda" % "2.3.0", + "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-base" % "2.3.0", + "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-json-provider" % "2.3.0", + "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-smile-provider" % "2.3.0", + "com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % "2.3.0", + "com.sun.jersey" % "jersey-servlet" % "1.17.1", + "mysql" % "mysql-connector-java" % "5.1.34", + "org.scalatest" %% "scalatest" % "2.2.3" % "test", + "org.mockito" % "mockito-core" % "1.10.19" % "test" +) + +assemblyMergeStrategy in assembly := { + case path if path contains "pom." => MergeStrategy.first + case path if path contains "javax.inject.Named" => MergeStrategy.first + case path if path contains "mime.types" => MergeStrategy.first + case path if path contains "org/apache/commons/logging/impl/SimpleLog.class" => MergeStrategy.first + case path if path contains "org/apache/commons/logging/impl/SimpleLog$1.class" => MergeStrategy.first + case path if path contains "org/apache/commons/logging/impl/NoOpLog.class" => MergeStrategy.first + case path if path contains "org/apache/commons/logging/LogFactory.class" => MergeStrategy.first + case path if path contains "org/apache/commons/logging/LogConfigurationException.class" => MergeStrategy.first + case path if path contains "org/apache/commons/logging/Log.class" => MergeStrategy.first + case path if path contains "META-INF/jersey-module-version" => MergeStrategy.first + case path if path contains ".properties" => MergeStrategy.first + case path if path contains ".class" => MergeStrategy.first + case x => + val oldStrategy = (assemblyMergeStrategy in assembly).value + oldStrategy(x) +} +```