mirror of https://github.com/apache/druid.git
Merge pull request #2597 from metamx/forwardPortMMX23
Forward port - Fix dependency problems
This commit is contained in:
commit
863aa66808
|
@ -145,6 +145,21 @@
|
|||
<groupId>com.lmax</groupId>
|
||||
<artifactId>disruptor</artifactId>
|
||||
</dependency>
|
||||
<!--
|
||||
Druid doesn't ACTUALLY depend on jets3t in its core, but quite a few of the extensions do. This leads to a nasty ClassLoader problem
|
||||
There's a bug in
|
||||
https://github.com/apache/httpclient/blob/4.5.2/httpclient/src/main/java-deprecated/org/apache/http/impl/client/AbstractHttpClient.java#L332
|
||||
Where httpclient does not care what your context classloader is when looking for the connection manager factories. See https://issues.apache.org/jira/browse/HTTPCLIENT-1727
|
||||
A few extensions depend on jets3t, so we include it explicitly here to make sure it can load up its
|
||||
org.jets3t.service.utils.RestUtils$ConnManagerFactory
|
||||
properly
|
||||
Future releases which include HTTPCLIENT-1727 should probably set the context loader whenever jets3t calls are used
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>net.java.dev.jets3t</groupId>
|
||||
<artifactId>jets3t</artifactId>
|
||||
<version>0.9.4</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
<dependency>
|
||||
|
|
|
@ -187,3 +187,40 @@ If you want your extension to be included, you can add your extension's maven co
|
|||
|
||||
During `mvn install`, maven will install your extension to the local maven repository, and then call [pull-deps](../operations/pull-deps.html) to pull your extension from
|
||||
there. In the end, you should see your extension underneath `distribution/target/extensions` and within Druid tarball.
|
||||
|
||||
### Managing dependencies
|
||||
Managing library collisions can be daunting for extensions which draw in commonly used libraries. Here is a list of group IDs for libraries that are suggested to be specified with a `provided` scope to prevent collision with versions used in druid:
|
||||
```
|
||||
"io.druid",
|
||||
"com.metamx.druid",
|
||||
"asm",
|
||||
"org.ow2.asm",
|
||||
"org.jboss.netty",
|
||||
"com.google.guava",
|
||||
"com.google.code.findbugs",
|
||||
"com.google.protobuf",
|
||||
"com.esotericsoftware.minlog",
|
||||
"log4j",
|
||||
"org.slf4j",
|
||||
"commons-logging",
|
||||
"org.eclipse.jetty",
|
||||
"org.mortbay.jetty",
|
||||
"com.sun.jersey",
|
||||
"com.sun.jersey.contribs",
|
||||
"common-beanutils",
|
||||
"commons-codec",
|
||||
"commons-lang",
|
||||
"commons-cli",
|
||||
"commons-io",
|
||||
"javax.activation",
|
||||
"org.apache.httpcomponents",
|
||||
"org.apache.zookeeper",
|
||||
"org.codehaus.jackson",
|
||||
"com.fasterxml.jackson",
|
||||
"com.fasterxml.jackson.core",
|
||||
"com.fasterxml.jackson.dataformat",
|
||||
"com.fasterxml.jackson.datatype",
|
||||
"org.roaringbitmap",
|
||||
"net.java.dev.jets3t"
|
||||
```
|
||||
See the documentation in `io.druid.cli.PullDependencies` for more information.
|
||||
|
|
|
@ -38,12 +38,27 @@
|
|||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.microsoft.azure</groupId>
|
||||
<artifactId>azure-storage</artifactId>
|
||||
<version>2.1.0</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-core</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
|
|
|
@ -37,15 +37,87 @@
|
|||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.netflix.astyanax</groupId>
|
||||
<artifactId>astyanax</artifactId>
|
||||
<version>1.0.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>jline</groupId>
|
||||
<artifactId>jline</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>joda-time</groupId>
|
||||
<artifactId>joda-time</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>jetty-util</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.ning</groupId>
|
||||
<artifactId>compress-lzf</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-cli</groupId>
|
||||
<artifactId>commons-cli</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-httpclient</groupId>
|
||||
<artifactId>commons-httpclient</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpcore</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-core-asl</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
|
|
|
@ -46,11 +46,18 @@
|
|||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.inject</groupId>
|
||||
<artifactId>guice</artifactId>
|
||||
<version>${guice.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>aopalliance</groupId>
|
||||
<artifactId>aopalliance</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<!--$NO-MVN-MAN-VER$ -->
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
@ -70,11 +77,27 @@
|
|||
<groupId>org.apache.jclouds.driver</groupId>
|
||||
<artifactId>jclouds-slf4j</artifactId>
|
||||
<version>${jclouds.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.ws.rs</groupId>
|
||||
<artifactId>jsr311-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.jclouds.driver</groupId>
|
||||
<artifactId>jclouds-sshj</artifactId>
|
||||
<version>${jclouds.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<!-- Rackspace US dependencies -->
|
||||
<dependency>
|
||||
|
|
|
@ -35,10 +35,12 @@
|
|||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.metamx</groupId>
|
||||
<artifactId>emitter</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
|
@ -53,6 +55,14 @@
|
|||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>net.jpountz.lz4</groupId>
|
||||
<artifactId>lz4</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<!-- Tests -->
|
||||
|
|
|
@ -37,6 +37,7 @@
|
|||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.rabbitmq</groupId>
|
||||
|
@ -47,6 +48,12 @@
|
|||
<groupId>net.jodah</groupId>
|
||||
<artifactId>lyra</artifactId>
|
||||
<version>0.3.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
|
|
|
@ -43,27 +43,96 @@
|
|||
<version>${project.parent.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<!-- override jets3t from hadoop-core -->
|
||||
<dependency>
|
||||
<groupId>net.java.dev.jets3t</groupId>
|
||||
<artifactId>jets3t</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<!-- override httpclient / httpcore version from jets3t -->
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpcore</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<scope>compile</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>commons-cli</groupId>
|
||||
<artifactId>commons-cli</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-httpclient</groupId>
|
||||
<artifactId>commons-httpclient</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpcore</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-core-asl</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.ws.rs</groupId>
|
||||
<artifactId>jsr311-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>jetty-util</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-annotations</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.activation</groupId>
|
||||
<artifactId>activation</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
<artifactId>jersey-core</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.metamx</groupId>
|
||||
|
|
|
@ -35,6 +35,7 @@
|
|||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-processing</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
|
|
|
@ -35,11 +35,30 @@
|
|||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka_2.10</artifactId>
|
||||
<version>0.8.2.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>net.jpountz.lz4</groupId>
|
||||
<artifactId>lz4</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
|
|
|
@ -37,11 +37,13 @@
|
|||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-processing</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.druid.extensions</groupId>
|
||||
|
@ -52,6 +54,7 @@
|
|||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-server</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
|
@ -66,6 +69,14 @@
|
|||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>net.jpountz.lz4</groupId>
|
||||
<artifactId>lz4</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
|
|
|
@ -37,16 +37,19 @@
|
|||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-processing</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.druid</groupId>
|
||||
<artifactId>druid-server</artifactId>
|
||||
<version>${project.parent.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
|
|
|
@ -45,29 +45,11 @@
|
|||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- override jets3t from hadoop-core -->
|
||||
<dependency>
|
||||
<groupId>net.java.dev.jets3t</groupId>
|
||||
<artifactId>jets3t</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<!-- jets3t requires log4j 1.2 compatability -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-1.2-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<!-- override httpclient / httpcore version from jets3t -->
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpcore</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.metamx</groupId>
|
||||
<artifactId>emitter</artifactId>
|
||||
|
|
|
@ -53,19 +53,23 @@
|
|||
<dependency>
|
||||
<groupId>net.java.dev.jets3t</groupId>
|
||||
<artifactId>jets3t</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- override httpclient / httpcore version from jets3t -->
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpcore</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
|
|
|
@ -22,7 +22,6 @@ package io.druid.indexing.common.task;
|
|||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Predicate;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.Collections2;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.inject.Injector;
|
||||
|
@ -132,23 +131,16 @@ public abstract class HadoopTask extends AbstractTask
|
|||
? hadoopDependencyCoordinates
|
||||
: toolbox.getConfig().getDefaultHadoopCoordinates();
|
||||
|
||||
final List<URL> extensionURLs = Lists.newArrayList();
|
||||
final List<URL> jobURLs = Lists.newArrayList(
|
||||
Arrays.asList(((URLClassLoader) HadoopIndexTask.class.getClassLoader()).getURLs())
|
||||
);
|
||||
|
||||
for (final File extension : Initialization.getExtensionFilesToLoad(extensionsConfig)) {
|
||||
final ClassLoader extensionLoader = Initialization.getClassLoaderForExtension(extension);
|
||||
extensionURLs.addAll(Arrays.asList(((URLClassLoader) extensionLoader).getURLs()));
|
||||
jobURLs.addAll(Arrays.asList(((URLClassLoader) extensionLoader).getURLs()));
|
||||
}
|
||||
|
||||
final List<URL> nonHadoopURLs = Lists.newArrayList();
|
||||
nonHadoopURLs.addAll(Arrays.asList(((URLClassLoader) HadoopIndexTask.class.getClassLoader()).getURLs()));
|
||||
|
||||
final List<URL> druidURLs = ImmutableList.copyOf(
|
||||
Collections2.filter(nonHadoopURLs, IS_DRUID_URL)
|
||||
);
|
||||
final List<URL> nonHadoopNotDruidURLs = Lists.newArrayList(nonHadoopURLs);
|
||||
nonHadoopNotDruidURLs.removeAll(druidURLs);
|
||||
|
||||
final List<URL> localClassLoaderURLs = new ArrayList<>();
|
||||
localClassLoaderURLs.addAll(nonHadoopNotDruidURLs);
|
||||
final List<URL> localClassLoaderURLs = new ArrayList<>(jobURLs);
|
||||
|
||||
// hadoop dependencies come before druid classes because some extensions depend on them
|
||||
for (final File hadoopDependency :
|
||||
|
@ -160,15 +152,12 @@ public abstract class HadoopTask extends AbstractTask
|
|||
localClassLoaderURLs.addAll(Arrays.asList(((URLClassLoader) hadoopLoader).getURLs()));
|
||||
}
|
||||
|
||||
localClassLoaderURLs.addAll(druidURLs);
|
||||
localClassLoaderURLs.addAll(extensionURLs);
|
||||
final ClassLoader classLoader = new URLClassLoader(localClassLoaderURLs.toArray(new URL[localClassLoaderURLs.size()]), null);
|
||||
final ClassLoader classLoader = new URLClassLoader(
|
||||
localClassLoaderURLs.toArray(new URL[localClassLoaderURLs.size()]),
|
||||
null
|
||||
);
|
||||
|
||||
final List<URL> jobUrls = Lists.newArrayList();
|
||||
jobUrls.addAll(nonHadoopURLs);
|
||||
jobUrls.addAll(extensionURLs);
|
||||
|
||||
System.setProperty("druid.hadoop.internal.classpath", Joiner.on(File.pathSeparator).join(jobUrls));
|
||||
System.setProperty("druid.hadoop.internal.classpath", Joiner.on(File.pathSeparator).join(jobURLs));
|
||||
return classLoader;
|
||||
}
|
||||
|
||||
|
|
50
pom.xml
50
pom.xml
|
@ -128,6 +128,12 @@
|
|||
<groupId>com.metamx</groupId>
|
||||
<artifactId>java-util</artifactId>
|
||||
<version>${metamx.java-util.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.metamx</groupId>
|
||||
|
@ -201,6 +207,12 @@
|
|||
<groupId>org.skife.config</groupId>
|
||||
<artifactId>config-magic</artifactId>
|
||||
<version>0.9</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
|
@ -466,6 +478,12 @@
|
|||
<groupId>io.tesla.aether</groupId>
|
||||
<artifactId>tesla-aether</artifactId>
|
||||
<version>0.0.5</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.aether</groupId>
|
||||
|
@ -476,6 +494,32 @@
|
|||
<groupId>net.java.dev.jets3t</groupId>
|
||||
<artifactId>jets3t</artifactId>
|
||||
<version>0.9.4</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpcore</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-core-asl</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<!-- The htttpcomponents artifacts have non-matching release cadence -->
|
||||
<dependency>
|
||||
|
@ -558,6 +602,12 @@
|
|||
<groupId>com.ircclouds.irc</groupId>
|
||||
<artifactId>irc-api</artifactId>
|
||||
<version>1.0-0014</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.maxmind.geoip2</groupId>
|
||||
|
|
|
@ -64,8 +64,7 @@ public class PullDependencies implements Runnable
|
|||
private static final Logger log = new Logger(PullDependencies.class);
|
||||
|
||||
private static final Set<String> exclusions = Sets.newHashSet(
|
||||
"io.druid",
|
||||
"com.metamx.druid",
|
||||
/*
|
||||
|
||||
// It is possible that extensions will pull down a lot of jars that are either
|
||||
// duplicates OR conflict with druid jars. In that case, there are two problems that arise
|
||||
|
@ -94,6 +93,14 @@ public class PullDependencies implements Runnable
|
|||
// Different tasks which are classloader sensitive attempt to maintain a sane order for loading libraries in the
|
||||
// classloader, but it is always possible that something didn't load in the right order. Also we don't want to be
|
||||
// throwing around a ton of jars we don't need to.
|
||||
//
|
||||
// Here is a list of dependencies extensions should probably exclude.
|
||||
//
|
||||
// Conflicts can be discovered using the following command on the distribution tarball:
|
||||
// `find lib -iname *.jar | cut -d / -f 2 | sed -e 's/-[0-9]\.[0-9]/@/' | cut -f 1 -d @ | sort | uniq | xargs -I {} find extensions -name "*{}*.jar" | sort`
|
||||
|
||||
"io.druid",
|
||||
"com.metamx.druid",
|
||||
"asm",
|
||||
"org.ow2.asm",
|
||||
"org.jboss.netty",
|
||||
|
@ -123,6 +130,7 @@ public class PullDependencies implements Runnable
|
|||
"com.fasterxml.jackson.datatype",
|
||||
"org.roaringbitmap",
|
||||
"net.java.dev.jets3t"
|
||||
*/
|
||||
);
|
||||
|
||||
private TeslaAether aether;
|
||||
|
@ -294,6 +302,19 @@ public class PullDependencies implements Runnable
|
|||
@Override
|
||||
public boolean accept(DependencyNode node, List<DependencyNode> parents)
|
||||
{
|
||||
String scope = node.getDependency().getScope();
|
||||
if(scope != null) {
|
||||
scope = scope.toLowerCase();
|
||||
if(scope.equals("provided")) {
|
||||
return false;
|
||||
}
|
||||
if(scope.equals("test")) {
|
||||
return false;
|
||||
}
|
||||
if(scope.equals("system")) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (accept(node.getArtifact())) {
|
||||
return false;
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue