Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-02-16 15:05:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix low sonatype findings (#17017)

Browse Source 
Fixed vulnerabilities
CVE-2021-26291 : Apache Maven is vulnerable to Man-in-the-Middle (MitM) attacks. Various
functions across several files, mentioned below, allow for custom repositories to use the
insecure HTTP protocol. An attacker can exploit this as part of a Man-in-the-Middle (MitM)
attack, taking over or impersonating a repository using the insecure HTTP protocol.
Unsuspecting users may then have the compromised repository defined as a dependency in
their Project Object Model (pom) file and download potentially malicious files from it.
Was fixed by removing outdated tesla-aether library containing vulnerable maven-settings (v3.1.1) package, pull-deps utility updated to use maven resolver instead.

sonatype-2020-0244 : The joni package is vulnerable to Man-in-the-Middle (MitM) attacks.
This project downloads dependencies over HTTP due to an insecure repository configuration
within the .pom file. Consequently, a MitM could intercept requests to the specified
repository and replace the requested dependencies with malicious versions, which can execute
arbitrary code from the application that was built with them.
Was fixed by upgrading joni package to recommended 2.1.34 version
This commit is contained in:
Misha 2024-09-16 12:40:25 +02:00 committed by GitHub
parent a8d15182a3
commit 6aad9b08dd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 494 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
indexing-service/pom.xml
View File

@ -189,10 +189,6 @@
<artifactId>datasketches-memory</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>net.thisptr</groupId>
<artifactId>jackson-jq</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
@ -203,10 +199,6 @@
<artifactId>commons-collections4</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.eclipse.aether</groupId>
<artifactId>aether-api</artifactId>
</dependency>
<!-- Tests -->
<dependency>
<groupId>junit</groupId>
@ -267,6 +259,11 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-api</artifactId>
<version>1.3.1</version>
</dependency>
</dependencies>
<profiles>

2
indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java
View File

@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import net.thisptr.jackson.jq.internal.misc.Lists;
import com.google.common.collect.Lists;
import org.apache.druid.client.indexing.SamplerResponse;
import org.apache.druid.client.indexing.SamplerResponse.SamplerResponseRow;
import org.apache.druid.data.input.InputFormat;

116
licenses.yaml
View File

@ -1837,33 +1837,38 @@ name: Apache Maven
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 3.1.1
version: 3.6.0
libraries:
- org.apache.maven: maven-aether-provider
- org.apache.maven: maven-model
- org.apache.maven: maven-model-builder
- org.apache.maven: maven-repository-metadata
- org.apache.maven: maven-settings
- org.apache.maven: maven-settings-builder
- org.apache.maven: maven-builder-support
notices:
- maven-aether-provider: |
Maven Aether Provider
Copyright 2001-2013 The Apache Software Foundation
- maven-model: |
Maven Model
Copyright 2001-2013 The Apache Software Foundation
- maven-model-builder: |
Maven Model Builder
Copyright 2001-2013 The Apache Software Foundation
- maven-repository-metadata: |
Maven Repository Metadata Model
Copyright 2001-2013 The Apache Software Foundation
- maven-settings: |
Maven Settings
Copyright 2001-2013 The Apache Software Foundation
- maven-settings-builder: |
Maven Settings Builder
Copyright 2001-2013 The Apache Software Foundation
Copyright 2001-2018 The Apache Software Foundation
- maven-builder-support: |
Maven Builder Support
Copyright 2001-2018 The Apache Software Foundation
---
name: Maven Artifact Resolver Provider
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 3.6.0
libraries:
- org.apache.maven: maven-resolver-provider
- org.apache.maven: maven-model
- org.apache.maven: maven-model-builder
notices:
- maven-resolver-provider: |
Maven Artifact Resolver Provider
Copyright 2001-2018 The Apache Software Foundation
- maven-model: |
Maven Model
Copyright 2001-2018 The Apache Software Foundation
- maven-model-builder: |
Maven Model Builder
Copyright 2001-2018 The Apache Software Foundation
---
name: Apache Maven Artifact
@ -1879,6 +1884,67 @@ notices:
Copyright 2001-2018 The Apache Software Foundation
---
name: Maven Artifact Resolver Connector Basic
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 1.3.1
libraries:
- org.apache.maven.resolver: maven-resolver-connector-basic
- org.apache.maven.resolver: maven-resolver-spi
- org.apache.maven.resolver: maven-resolver-api
- org.apache.maven.resolver: maven-resolver-util
notices:
- maven-resolver-connector-basic: |
Maven Artifact Resolver Connector Basic
Copyright 2001-2018 The Apache Software Foundation
- maven-resolver-spi: |
Maven Artifact Resolver SPI
Copyright 2001-2018 The Apache Software Foundation
- maven-resolver-api: |
Maven Artifact Resolver API
Copyright 2001-2018 The Apache Software Foundation
- maven-resolver-util: |
Maven Artifact Resolver Utilities
Copyright 2001-2018 The Apache Software Foundation
---
name: Maven Artifact Resolver Transport HTTP
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 1.3.1
libraries:
- org.apache.maven.resolver: maven-resolver-transport-http
notices:
- maven-resolver-transport-http: |
Maven Artifact Resolver Transport HTTP
Copyright 2001-2018 The Apache Software Foundation
---
name: Maven Artifact Resolver Implementation
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 1.3.1
libraries:
- org.apache.maven.resolver: maven-resolver-impl
notices:
- maven-resolver-impl: |
Maven Artifact Resolver Implementation
Copyright 2001-2018 The Apache Software Foundation
---
name: Plexus Component Annotations
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 1.7.1
libraries:
- org.codehaus.plexus: plexus-component-annotations
---
name: Apache Maven Wagon API
license_category: binary
module: java-core
@ -1967,7 +2033,7 @@ name: Plexus Interpolation API
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 1.19
version: 1.25
libraries:
- org.codehaus.plexus: plexus-interpolation
@ -3245,7 +3311,7 @@ name: JCodings
license_category: binary
module: java-core
license_name: MIT License
version: 1.0.43
version: 1.0.50
copyright: JRuby Team
license_file_path: licenses/bin/jcodings.MIT
libraries:
@ -3257,7 +3323,7 @@ name: Joni
license_category: binary
module: java-core
license_name: MIT License
version: 2.1.27
version: 2.1.34
copyright: JRuby Team
license_file_path: licenses/bin/joni.MIT
libraries:

5
pom.xml
View File

@ -1363,6 +1363,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jruby.joni</groupId>
<artifactId>joni</artifactId>
<version>2.1.34</version>
</dependency>
</dependencies>
</dependencyManagement>

19
server/pom.xml
View File

@ -140,8 +140,19 @@
<artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>io.tesla.aether</groupId>
<artifactId>tesla-aether</artifactId>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-connector-basic</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-transport-http</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-resolver-provider</artifactId>
<version>3.6.0</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
@ -500,7 +511,9 @@
<configuration>
<usedDependencies>
<!-- These are needed for scope: compile -->
<dependency>io.tesla.aether:tesla-aether</dependency>
<dependency>org.apache.maven:maven-resolver-provider</dependency>
<dependency>org.apache.maven.resolver:maven-resolver-transport-http</dependency>
<dependency>org.apache.maven.resolver:maven-resolver-connector-basic</dependency>
<!-- These are needed for scope: runtime -->
<dependency>org.xerial.snappy:snappy-java</dependency>
</usedDependencies>

54
services/pom.xml
View File

@ -56,6 +56,12 @@
<groupId>org.apache.druid</groupId>
<artifactId>druid-indexing-service</artifactId>
<version>${project.parent.version}</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.aether</groupId>
<artifactId>aether-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
@ -171,10 +177,6 @@
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.aether</groupId>
<artifactId>aether-api</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
@ -187,17 +189,49 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.aether</groupId>
<artifactId>aether-util</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
</dependency>
<dependency>
<groupId>io.tesla.aether</groupId>
<artifactId>tesla-aether</artifactId>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-connector-basic</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-transport-http</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-util</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-impl</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-spi</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven.resolver</groupId>
<artifactId>maven-resolver-api</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-artifact</artifactId>
<version>3.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-resolver-provider</artifactId>
<version>3.6.0</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>

360
services/src/main/java/org/apache/druid/cli/PullDependencies.java
View File

@ -22,43 +22,45 @@ package org.apache.druid.cli;
import com.github.rvesse.airline.annotations.Command;
import com.github.rvesse.airline.annotations.Option;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.SetMultimap;
import com.google.inject.Inject;
import io.netty.util.SuppressForbidden;
import io.tesla.aether.Repository;
import io.tesla.aether.TeslaAether;
import io.tesla.aether.guice.RepositorySystemSessionProvider;
import io.tesla.aether.internal.DefaultTeslaAether;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.java.util.common.FileUtils;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.maven.artifact.resolver.ArtifactNotFoundException;
import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
import org.eclipse.aether.DefaultRepositorySystemSession;
import org.eclipse.aether.RepositorySystem;
import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.artifact.Artifact;
import org.eclipse.aether.artifact.DefaultArtifact;
import org.eclipse.aether.collection.CollectRequest;
import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
import org.eclipse.aether.graph.Dependency;
import org.eclipse.aether.graph.DependencyNode;
import org.eclipse.aether.repository.Authentication;
import org.eclipse.aether.impl.DefaultServiceLocator;
import org.eclipse.aether.repository.LocalRepository;
import org.eclipse.aether.repository.Proxy;
import org.eclipse.aether.repository.RemoteRepository;
import org.eclipse.aether.resolution.ArtifactResult;
import org.eclipse.aether.resolution.DependencyRequest;
import org.eclipse.aether.resolution.DependencyResolutionException;
import org.eclipse.aether.resolution.DependencyResult;
import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
import org.eclipse.aether.spi.connector.transport.TransporterFactory;
import org.eclipse.aether.transport.http.HttpTransporterFactory;
import org.eclipse.aether.util.artifact.JavaScopes;
import org.eclipse.aether.util.filter.DependencyFilterUtils;
import org.eclipse.aether.util.repository.AuthenticationBuilder;
import org.eclipse.aether.util.repository.DefaultProxySelector;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@ -86,72 +88,6 @@ public class PullDependencies implements Runnable
.put("com.fasterxml.jackson.core", "jackson-core")
.put("com.fasterxml.jackson.core", "jackson-annotations")
.build();
/*
It is possible that extensions will pull down a lot of jars that are either
duplicates OR conflict with druid jars. In that case, there are two problems that arise
1. Large quantity of jars are passed around to things like hadoop when they are not needed (and should not be included)
2. Classpath priority becomes "mostly correct" and attempted to enforced correctly, but not fully tested
These jar groups should be included by druid and *not* pulled down in extensions
Note to future developers: This list is hand-crafted and will probably be out of date in the future
A good way to know where to look for errant dependencies is to compare the lib/ directory in the distribution
tarball with the jars included in the extension directories.
This list is best-effort, and might still pull down more than desired.
A simple example is that if an extension's dependency uses some-library-123.jar,
druid uses some-library-456.jar, and hadoop uses some-library-666.jar, then we probably want to use some-library-456.jar,
so don't pull down some-library-123.jar, and ask hadoop to load some-library-456.jar.
In the case where some-library is NOT on this list, both some-library-456.jar and some-library-123.jar will be
on the class path and propagated around the system. Most places TRY to make sure some-library-456.jar has
precedence, but it is easy for this assumption to be violated and for the precedence of some-library-456.jar,
some-library-123.jar and some-library-456.jar to not be properly defined.
As of this writing there are no special unit tests for classloader issues and library version conflicts.
Different tasks which are classloader sensitive attempt to maintain a sane order for loading libraries in the
classloader, but it is always possible that something didn't load in the right order. Also we don't want to be
throwing around a ton of jars we don't need to.
Here is a list of dependencies extensions should probably exclude.
Conflicts can be discovered using the following command on the distribution tarball:
`find lib -iname "*.jar" | cut -d / -f 2 | sed -e 's/-[0-9]\.[0-9]/@/' | cut -f 1 -d @ | sort | uniq | xargs -I {} find extensions -name "*{}*.jar" | sort`
"org.apache.druid",
"com.metamx.druid",
"asm",
"org.ow2.asm",
"org.jboss.netty",
"com.google.guava",
"com.google.code.findbugs",
"com.google.protobuf",
"com.esotericsoftware.minlog",
"log4j",
"org.slf4j",
"commons-logging",
"org.eclipse.jetty",
"org.mortbay.jetty",
"com.sun.jersey",
"com.sun.jersey.contribs",
"common-beanutils",
"commons-codec",
"commons-lang",
"commons-cli",
"commons-io",
"javax.activation",
"org.apache.httpcomponents",
"org.apache.zookeeper",
"org.codehaus.jackson",
"com.fasterxml.jackson",
"com.fasterxml.jackson.core",
"com.fasterxml.jackson.dataformat",
"com.fasterxml.jackson.datatype",
"org.roaringbitmap",
"net.java.dev.jets3t"
*/
private static final Dependencies SECURITY_VULNERABILITY_EXCLUSIONS =
Dependencies.builder()
@ -160,8 +96,6 @@ public class PullDependencies implements Runnable
private final Dependencies hadoopExclusions;
private TeslaAether aether;
@Inject
public ExtensionsConfig extensionsConfig;
@ -196,60 +130,53 @@ public class PullDependencies implements Runnable
title = "A local repository that Maven will use to put downloaded files. Then pull-deps will lay these files out into the extensions directory as needed."
)
public String localRepository = StringUtils.format("%s/%s", System.getProperty("user.home"), ".m2/repository");
@Option(
name = {"-r", "--remoteRepository"},
title = "Add a remote repository. Unless --no-default-remote-repositories is provided, these will be used after https://repo1.maven.org/maven2/"
)
List<String> remoteRepositories = new ArrayList<>();
@Option(
name = "--no-default-remote-repositories",
description = "Don't use the default remote repositories, only use the repositories provided directly via --remoteRepository"
)
public boolean noDefaultRemoteRepositories = false;
@Option(
name = {"-d", "--defaultVersion"},
title = "Version to use for extension artifacts without version information."
)
public String defaultVersion = PullDependencies.class.getPackage().getImplementationVersion();
@Option(
name = {"--use-proxy"},
title = "Use http/https proxy to pull dependencies."
)
public boolean useProxy = false;
@Option(
name = {"--proxy-type"},
title = "The proxy type, should be either http or https"
)
public String proxyType = "https";
@Option(
name = {"--proxy-host"},
title = "The proxy host"
)
public String proxyHost = "";
@Option(
name = {"--proxy-port"},
title = "The proxy port"
)
public int proxyPort = -1;
@Option(
name = {"--proxy-username"},
title = "The proxy username"
)
public String proxyUsername = "";
@Option(
name = {"--proxy-password"},
title = "The proxy password"
)
public String proxyPassword = "";
@Option(
name = {"-r", "--remoteRepository"},
title = "Add a remote repository. Unless --no-default-remote-repositories is provided, these will be used after https://repo1.maven.org/maven2/"
)
List<String> remoteRepositories = new ArrayList<>();
private RepositorySystem repositorySystem;
private RepositorySystemSession repositorySystemSession;
@SuppressWarnings("unused") // used by com.github.rvesse.airline
public PullDependencies()
@ -261,18 +188,74 @@ public class PullDependencies implements Runnable
}
// Used for testing only
PullDependencies(TeslaAether aether, ExtensionsConfig extensionsConfig, Dependencies hadoopExclusions)
PullDependencies(
RepositorySystem repositorySystem,
RepositorySystemSession repositorySystemSession,
ExtensionsConfig extensionsConfig,
Dependencies hadoopExclusions
)
{
this.aether = aether;
this.repositorySystem = repositorySystem;
this.repositorySystemSession = repositorySystemSession;
this.extensionsConfig = extensionsConfig;
this.hadoopExclusions = hadoopExclusions;
}
private RepositorySystem getRepositorySystem()
{
DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
return locator.getService(RepositorySystem.class);
}
protected RepositorySystemSession getRepositorySystemSession()
{
DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession();
LocalRepository localRepo = new LocalRepository(localRepository);
session.setLocalRepositoryManager(repositorySystem.newLocalRepositoryManager(session, localRepo));
// Set up the proxy configuration if required
if (useProxy) {
Proxy proxy = new Proxy(
proxyType,
proxyHost,
proxyPort,
isBlank(proxyUsername) ? null : new AuthenticationBuilder()
.addUsername(proxyUsername)
.addPassword(proxyPassword)
.build()
);
final DefaultProxySelector proxySelector = new DefaultProxySelector();
proxySelector.add(proxy, null);
session.setProxySelector(proxySelector);
}
return session;
}
protected List<RemoteRepository> getRemoteRepositories()
{
List<RemoteRepository> repositories = new ArrayList<>();
if (!noDefaultRemoteRepositories) {
repositories.add(new RemoteRepository.Builder("central", "default", DEFAULT_REMOTE_REPOSITORIES.get(0)).build());
}
for (String repoUrl : remoteRepositories) {
repositories.add(new RemoteRepository.Builder(null, "default", repoUrl).build());
}
return repositories;
}
@Override
public void run()
{
if (aether == null) {
aether = getAetherClient();
if (repositorySystem == null) {
repositorySystem = getRepositorySystem();
}
final File extensionsDir = new File(extensionsConfig.getDirectory());
@ -334,7 +317,7 @@ public class PullDependencies implements Runnable
}
}
private Artifact getArtifact(String coordinate)
protected Artifact getArtifact(String coordinate)
{
DefaultArtifact versionedArtifact;
try {
@ -367,6 +350,12 @@ public class PullDependencies implements Runnable
{
final CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new Dependency(versionedArtifact, JavaScopes.RUNTIME));
List<RemoteRepository> repositories = getRemoteRepositories();
for (RemoteRepository repo : repositories) {
collectRequest.addRepository(repo);
}
final DependencyRequest dependencyRequest = new DependencyRequest(
collectRequest,
DependencyFilterUtils.andFilter(
@ -375,13 +364,7 @@ public class PullDependencies implements Runnable
String scope = node.getDependency().getScope();
if (scope != null) {
scope = StringUtils.toLowerCase(scope);
if ("provided".equals(scope)) {
return false;
}
if ("test".equals(scope)) {
return false;
}
if ("system".equals(scope)) {
if ("provided".equals(scope) || "test".equals(scope) || "system".equals(scope)) {
return false;
}
}
@ -402,7 +385,17 @@ public class PullDependencies implements Runnable
try {
log.info("Start downloading extension [%s]", versionedArtifact);
final List<Artifact> artifacts = aether.resolveArtifacts(dependencyRequest);
if (repositorySystemSession == null) {
repositorySystemSession = getRepositorySystemSession();
}
final DependencyResult result = repositorySystem.resolveDependencies(
repositorySystemSession,
dependencyRequest
);
final List<Artifact> artifacts = result.getArtifactResults().stream()
.map(ArtifactResult::getArtifact)
.collect(Collectors.toList());
for (Artifact artifact : artifacts) {
if (exclusions.contain(artifact)) {
@ -413,140 +406,20 @@ public class PullDependencies implements Runnable
}
}
}
catch (Exception e) {
log.error(e, "Unable to resolve artifacts for [%s].", dependencyRequest);
catch (DependencyResolutionException e) {
if (e.getCause() instanceof ArtifactNotFoundException) {
log.error("Artifact not found in any configured repositories: [%s]", versionedArtifact);
} else {
log.error(e, "Unable to resolve artifacts for [%s].", dependencyRequest);
}
}
catch (IOException e) {
log.error(e, "I/O error while processing artifact [%s].", versionedArtifact);
throw new RuntimeException(e);
}
log.info("Finish downloading extension [%s]", versionedArtifact);
}
@SuppressForbidden(reason = "System#out")
private DefaultTeslaAether getAetherClient()
{
/*
DefaultTeslaAether logs a bunch of stuff to System.out, which is annoying. We choose to disable that
unless debug logging is turned on. "Disabling" it, however, is kinda bass-ackwards. We copy out a reference
to the current System.out, and set System.out to a noop output stream. Then after DefaultTeslaAether has pulled
The reference we swap things back.
This has implications for other things that are running in parallel to this. Namely, if anything else also grabs
a reference to System.out or tries to log to it while we have things adjusted like this, then they will also log
to nothingness. Fortunately, the code that calls this is single-threaded and shouldn't hopefully be running
alongside anything else that's grabbing System.out. But who knows.
*/
final List<String> remoteUriList = new ArrayList<>();
if (!noDefaultRemoteRepositories) {
remoteUriList.addAll(DEFAULT_REMOTE_REPOSITORIES);
}
remoteUriList.addAll(remoteRepositories);
List<Repository> remoteRepositories = new ArrayList<>();
for (String uri : remoteUriList) {
try {
URI u = new URI(uri);
Repository r = new Repository(uri);
if (u.getUserInfo() != null) {
String[] auth = u.getUserInfo().split(":", 2);
if (auth.length == 2) {
r.setUsername(auth[0]);
r.setPassword(auth[1]);
} else {
log.warn(
"Invalid credentials in repository URI, expecting [<user>:<password>], got [%s] for [%s]",
u.getUserInfo(),
uri
);
}
}
remoteRepositories.add(r);
}
catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
if (log.isTraceEnabled() || log.isDebugEnabled()) {
return createTeslaAether(remoteRepositories);
}
PrintStream oldOut = System.out;
try {
System.setOut(
new PrintStream(
new OutputStream()
{
@Override
public void write(int b)
{
}
@Override
public void write(byte[] b)
{
}
@Override
public void write(byte[] b, int off, int len)
{
}
},
false,
StringUtils.UTF8_STRING
)
);
return createTeslaAether(remoteRepositories);
}
catch (UnsupportedEncodingException e) {
// should never happen
throw new IllegalStateException(e);
}
finally {
System.setOut(oldOut);
}
}
private DefaultTeslaAether createTeslaAether(List<Repository> remoteRepositories)
{
if (!useProxy) {
return new DefaultTeslaAether(
localRepository,
remoteRepositories.toArray(new Repository[0])
);
}
if (!StringUtils.toLowerCase(proxyType).equals(Proxy.TYPE_HTTP) &&
!StringUtils.toLowerCase(proxyType).equals(Proxy.TYPE_HTTPS)) {
throw new IllegalArgumentException("invalid proxy type: " + proxyType);
}
RepositorySystemSession repositorySystemSession =
new RepositorySystemSessionProvider(new File(localRepository)).get();
List<RemoteRepository> rl = remoteRepositories.stream().map(r -> {
RemoteRepository.Builder builder = new RemoteRepository.Builder(r.getId(), "default", r.getUrl());
if (r.getUsername() != null && r.getPassword() != null) {
Authentication auth = new AuthenticationBuilder().addUsername(r.getUsername())
.addPassword(r.getPassword())
.build();
builder.setAuthentication(auth);
}
final Authentication proxyAuth;
if (Strings.isNullOrEmpty(proxyUsername)) {
proxyAuth = null;
} else {
proxyAuth = new AuthenticationBuilder().addUsername(proxyUsername).addPassword(proxyPassword).build();
}
builder.setProxy(new Proxy(proxyType, proxyHost, proxyPort, proxyAuth));
return builder.build();
}).collect(Collectors.toList());
return new DefaultTeslaAether(rl, repositorySystemSession);
}
/**
* Create the extension directory for a specific maven coordinate.
* The name of this directory should be the artifactId in the coordinate
@ -567,6 +440,11 @@ public class PullDependencies implements Runnable
}
}
private boolean isBlank(final String toCheck)
{
return toCheck == null || toCheck.isEmpty();
}
@VisibleForTesting
static class Dependencies
{
@ -579,17 +457,17 @@ public class PullDependencies implements Runnable
groupIdToArtifactIds = builder.groupIdToArtifactIdsBuilder.build();
}
static Builder builder()
{
return new Builder();
}
boolean contain(Artifact artifact)
{
Set<String> artifactIds = groupIdToArtifactIds.get(artifact.getGroupId());
return artifactIds.contains(ANY_ARTIFACT_ID) || artifactIds.contains(artifact.getArtifactId());
}
static Builder builder()
{
return new Builder();
}
static final class Builder
{
private final ImmutableSetMultimap.Builder<String, String> groupIdToArtifactIdsBuilder =

247
services/src/test/java/org/apache/druid/cli/PullDependenciesTest.java
View File

@ -21,16 +21,32 @@ package org.apache.druid.cli;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import io.tesla.aether.internal.DefaultTeslaAether;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
import org.eclipse.aether.DefaultRepositorySystemSession;
import org.eclipse.aether.RepositorySystem;
import org.eclipse.aether.RepositorySystemSession;
import org.eclipse.aether.artifact.Artifact;
import org.eclipse.aether.artifact.DefaultArtifact;
import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
import org.eclipse.aether.graph.DefaultDependencyNode;
import org.eclipse.aether.graph.Dependency;
import org.eclipse.aether.graph.DependencyFilter;
import org.eclipse.aether.graph.DependencyNode;
import org.eclipse.aether.impl.DefaultServiceLocator;
import org.eclipse.aether.repository.Authentication;
import org.eclipse.aether.repository.LocalRepository;
import org.eclipse.aether.repository.Proxy;
import org.eclipse.aether.repository.RemoteRepository;
import org.eclipse.aether.resolution.ArtifactRequest;
import org.eclipse.aether.resolution.ArtifactResult;
import org.eclipse.aether.resolution.DependencyRequest;
import org.eclipse.aether.resolution.DependencyResult;
import org.eclipse.aether.spi.connector.RepositoryConnectorFactory;
import org.eclipse.aether.spi.connector.transport.TransporterFactory;
import org.eclipse.aether.transport.http.HttpTransporterFactory;
import org.eclipse.aether.util.artifact.JavaScopes;
import org.eclipse.aether.util.repository.AuthenticationBuilder;
import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Before;
@ -50,6 +66,11 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.spy;
public class PullDependenciesTest
{
private static final String EXTENSION_A_COORDINATE = "groupX:extension_A:123";
@ -71,23 +92,18 @@ public class PullDependenciesTest
.put(DEPENDENCY_GROUPID, HADOOP_CLIENT_VULNERABLE_ARTIFACTID1)
.put(DEPENDENCY_GROUPID, HADOOP_CLIENT_VULNERABLE_ARTIFACTID2)
.build();
private static File localRepo; // a mock local repository that stores jars
private static Map<Artifact, List<String>> extensionToDependency;
@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
private File localRepo; // a mock local repository that stores jars
private final Artifact extension_A = new DefaultArtifact(EXTENSION_A_COORDINATE);
private final Artifact extension_B = new DefaultArtifact(EXTENSION_B_COORDINATE);
private final Artifact hadoop_client_2_3_0 = new DefaultArtifact(HADOOP_CLIENT_2_3_0_COORDINATE);
private final Artifact hadoop_client_2_4_0 = new DefaultArtifact(HADOOP_CLIENT_2_4_0_COORDINATE);
private PullDependencies pullDependencies;
private File rootExtensionsDir;
private File rootHadoopDependenciesDir;
private Map<Artifact, List<String>> extensionToDependency;
@Before
public void setUp() throws Exception
{
@ -105,18 +121,22 @@ public class PullDependenciesTest
rootExtensionsDir = temporaryFolder.newFolder("extensions");
rootHadoopDependenciesDir = temporaryFolder.newFolder("druid_hadoop_dependencies");
RepositorySystem realRepositorySystem = RealRepositorySystemUtil.newRepositorySystem();
RepositorySystem spyMockRepositorySystem = spy(realRepositorySystem);
RepositorySystemSession repositorySystemSession = RealRepositorySystemUtil.newRepositorySystemSession(
spyMockRepositorySystem,
localRepo.getPath()
);
doAnswer(invocation -> {
DependencyRequest request = invocation.getArgument(1);
return mockDependencyResult(request.getCollectRequest().getRoot().getArtifact());
}).when(spyMockRepositorySystem).resolveDependencies(eq(repositorySystemSession), any(DependencyRequest.class));
pullDependencies = new PullDependencies(
new DefaultTeslaAether()
{
@Override
public List<Artifact> resolveArtifacts(DependencyRequest request)
{
return getArtifactsForExtension(
request.getCollectRequest().getRoot().getArtifact(),
request.getFilter()
);
}
},
spyMockRepositorySystem,
repositorySystemSession,
new ExtensionsConfig()
{
@Override
@ -140,14 +160,15 @@ public class PullDependenciesTest
HADOOP_CLIENT_2_4_0_COORDINATE
);
// Because --clean is specified, pull-deps will first remove existing root extensions and hadoop dependencies
pullDependencies.clean = true;
}
private List<Artifact> getArtifactsForExtension(Artifact artifact, DependencyFilter filter)
private DependencyResult mockDependencyResult(Artifact artifact)
{
final List<String> names = extensionToDependency.get(artifact);
final List<Artifact> artifacts = new ArrayList<>();
final List<String> names = extensionToDependency.getOrDefault(artifact, Collections.emptyList());
final List<ArtifactResult> artifacts = new ArrayList<>();
List<DependencyNode> children = new ArrayList<>();
for (String name : names) {
final File jarFile = new File(localRepo, name + ".jar");
try {
@ -156,18 +177,23 @@ public class PullDependenciesTest
catch (IOException e) {
throw new RuntimeException(e);
}
DependencyNode node = new DefaultDependencyNode(
new Dependency(
new DefaultArtifact(DEPENDENCY_GROUPID, name, null, "jar", "1.0", null, jarFile),
"compile"
)
Artifact depArtifact = new DefaultArtifact("groupid", name, null, "jar", "1.0",
null, jarFile
);
if (filter.accept(node, Collections.emptyList())) {
artifacts.add(node.getArtifact());
}
DependencyNode depNode = new DefaultDependencyNode(new Dependency(depArtifact, JavaScopes.COMPILE));
children.add(depNode);
ArtifactResult artifactResult = new ArtifactResult(new ArtifactRequest(depNode));
artifactResult.setArtifact(depArtifact);
artifacts.add(artifactResult);
}
return artifacts;
DependencyNode rootNode = new DefaultDependencyNode(new Dependency(artifact, JavaScopes.COMPILE));
rootNode.setChildren(children);
DependencyResult result = new DependencyResult(new DependencyRequest());
result.setRoot(rootNode);
result.setArtifactResults(artifacts);
return result;
}
private List<File> getExpectedJarFiles(Artifact artifact)
@ -299,4 +325,157 @@ public class PullDependenciesTest
Assert.assertThat(dependencies, CoreMatchers.not(CoreMatchers.hasItem(HADOOP_CLIENT_VULNERABLE_JAR1)));
Assert.assertThat(dependencies, CoreMatchers.not(CoreMatchers.hasItem(HADOOP_CLIENT_VULNERABLE_JAR2)));
}
@Test
public void testPullDependenciesCleanFlag() throws IOException
{
File dummyFile1 = new File(rootExtensionsDir, "dummy.txt");
File dummyFile2 = new File(rootHadoopDependenciesDir, "dummy.txt");
Assert.assertTrue(dummyFile1.createNewFile());
Assert.assertTrue(dummyFile2.createNewFile());
pullDependencies.clean = true;
pullDependencies.run();
Assert.assertFalse(dummyFile1.exists());
Assert.assertFalse(dummyFile2.exists());
}
@Test
public void testPullDependenciesNoDefaultRemoteRepositories()
{
pullDependencies.noDefaultRemoteRepositories = true;
pullDependencies.remoteRepositories = ImmutableList.of("https://custom.repo");
pullDependencies.run();
List<RemoteRepository> repositories = pullDependencies.getRemoteRepositories();
Assert.assertEquals(1, repositories.size());
Assert.assertEquals("https://custom.repo", repositories.get(0).getUrl());
}
@Test
public void testPullDependenciesDirectoryCreationFailure() throws IOException
{
if (rootExtensionsDir.exists()) {
rootExtensionsDir.delete();
}
Assert.assertTrue(rootExtensionsDir.createNewFile());
Assert.assertThrows(IllegalArgumentException.class, () -> pullDependencies.run());
}
@Test
public void testGetArtifactWithValidCoordinate()
{
String coordinate = "groupX:artifactX:1.0.0";
DefaultArtifact artifact = (DefaultArtifact) pullDependencies.getArtifact(coordinate);
Assert.assertEquals("groupX", artifact.getGroupId());
Assert.assertEquals("artifactX", artifact.getArtifactId());
Assert.assertEquals("1.0.0", artifact.getVersion());
}
@Test
public void testGetArtifactwithCoordinateWithoutDefaultVersion()
{
String coordinate = "groupY:artifactY";
Assert.assertThrows(
"Bad artifact coordinates groupY:artifactY, expected format is <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>",
IllegalArgumentException.class,
() -> pullDependencies.getArtifact(coordinate)
);
}
@Test
public void testGetArtifactWithCoordinateWithoutVersion()
{
pullDependencies.defaultVersion = "2.0.0";
String coordinate = "groupY:artifactY";
DefaultArtifact artifact = (DefaultArtifact) pullDependencies.getArtifact(coordinate);
Assert.assertEquals("groupY", artifact.getGroupId());
Assert.assertEquals("artifactY", artifact.getArtifactId());
Assert.assertEquals("2.0.0", artifact.getVersion());
}
@Test
public void testGetRemoteRepositoriesWithDefaultRepositories()
{
pullDependencies.noDefaultRemoteRepositories = false; // Use default remote repositories
pullDependencies.remoteRepositories = ImmutableList.of("https://custom.repo");
List<RemoteRepository> repositories = pullDependencies.getRemoteRepositories();
Assert.assertEquals(2, repositories.size());
Assert.assertEquals("https://repo1.maven.org/maven2/", repositories.get(0).getUrl());
Assert.assertEquals("https://custom.repo", repositories.get(1).getUrl());
}
@Test
public void testGetRepositorySystemSessionWithProxyConfiguration()
{
pullDependencies.useProxy = true;
pullDependencies.proxyType = "http";
pullDependencies.proxyHost = "localhost";
pullDependencies.proxyPort = 8080;
pullDependencies.proxyUsername = "user";
pullDependencies.proxyPassword = "password";
DefaultRepositorySystemSession session = (DefaultRepositorySystemSession) pullDependencies.getRepositorySystemSession();
LocalRepository localRepo = session.getLocalRepositoryManager().getRepository();
Assert.assertEquals(pullDependencies.localRepository, localRepo.getBasedir().getAbsolutePath());
Proxy proxy = session.getProxySelector().getProxy(
new RemoteRepository.Builder("test", "default", "http://example.com").build()
);
RemoteRepository testRepository = new RemoteRepository.Builder("test", "default", "http://example.com")
.setProxy(proxy)
.build();
Assert.assertNotNull(proxy);
Assert.assertEquals("localhost", proxy.getHost());
Assert.assertEquals(8080, proxy.getPort());
Assert.assertEquals("http", proxy.getType());
Authentication auth = new AuthenticationBuilder().addUsername("user").addPassword("password").build();
Assert.assertEquals(auth, proxy.getAuthentication());
}
@Test
public void testGetRepositorySystemSessionWithoutProxyConfiguration()
{
pullDependencies.useProxy = false;
DefaultRepositorySystemSession session = (DefaultRepositorySystemSession) pullDependencies.getRepositorySystemSession();
LocalRepository localRepo = session.getLocalRepositoryManager().getRepository();
Assert.assertEquals(pullDependencies.localRepository, localRepo.getBasedir().getAbsolutePath());
Proxy proxy = session.getProxySelector().getProxy(
new RemoteRepository.Builder("test", "default", "http://example.com").build()
);
Assert.assertNull(proxy);
}
private static class RealRepositorySystemUtil
{
public static RepositorySystem newRepositorySystem()
{
DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator();
locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class);
locator.addService(TransporterFactory.class, HttpTransporterFactory.class);
return locator.getService(RepositorySystem.class);
}
public static DefaultRepositorySystemSession newRepositorySystemSession(
RepositorySystem system,
String localRepoPath
)
{
DefaultRepositorySystemSession session = MavenRepositorySystemUtils.newSession();
LocalRepository localRepo = new LocalRepository(localRepoPath);
session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, localRepo));
return session;
}
}
}