From b084075279be516df34147d22d97bdc030961a6e Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 8 Mar 2018 07:05:43 +0800 Subject: [PATCH] Add http/https proxy options to PullDependencies.java (#5450) --- docs/content/operations/pull-deps.md | 24 +++++ .../java/io/druid/cli/PullDependencies.java | 93 +++++++++++++++++-- 2 files changed, 109 insertions(+), 8 deletions(-) diff --git a/docs/content/operations/pull-deps.md b/docs/content/operations/pull-deps.md index 16a246deb37..e2444d4e787 100644 --- a/docs/content/operations/pull-deps.md +++ b/docs/content/operations/pull-deps.md @@ -40,6 +40,30 @@ Don't use the default remote repositories, only use the repositories provided di Version to use for extension coordinate that doesn't have a version information. For example, if extension coordinate is `io.druid.extensions:mysql-metadata-storage`, and default version is `#{DRUIDVERSION}`, then this coordinate will be treated as `io.druid.extensions:mysql-metadata-storage:#{DRUIDVERSION}` +`--use-proxy` + +Use http/https proxy to send request to the remote repository servers. `--proxy-host` and `--proxy-port` must be set explicitly if this option is enabled. + +`--proxy-type` + +Set the proxy type, Should be either *http* or *https*, default value is *https*. + +`--proxy-host` + +Set the proxy host. e.g. proxy.com. + +`--proxy-port` + +Set the proxy port number. e.g. 8080. + +`--proxy-username` + +Set a username to connect to the proxy, this option is only required if the proxy server uses authentication. + +`--proxy-password` + +Set a password to connect to the proxy, this option is only required if the proxy server uses authentication. + To run `pull-deps`, you should 1) Specify `druid.extensions.directory` and `druid.extensions.hadoopDependenciesDir`, these two properties tell `pull-deps` where to put extensions. If you don't specify them, default values will be used, see [Configuration](../configuration/index.html). diff --git a/services/src/main/java/io/druid/cli/PullDependencies.java b/services/src/main/java/io/druid/cli/PullDependencies.java index 646512f5304..5b9c6509f94 100644 --- a/services/src/main/java/io/druid/cli/PullDependencies.java +++ b/services/src/main/java/io/druid/cli/PullDependencies.java @@ -19,6 +19,7 @@ package io.druid.cli; +import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -33,17 +34,23 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.tesla.aether.Repository; import io.tesla.aether.TeslaAether; +import io.tesla.aether.guice.RepositorySystemSessionProvider; import io.tesla.aether.internal.DefaultTeslaAether; import org.apache.commons.io.FileUtils; +import org.eclipse.aether.RepositorySystemSession; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.artifact.DefaultArtifact; import org.eclipse.aether.collection.CollectRequest; import org.eclipse.aether.graph.Dependency; import org.eclipse.aether.graph.DependencyFilter; import org.eclipse.aether.graph.DependencyNode; +import org.eclipse.aether.repository.Authentication; +import org.eclipse.aether.repository.Proxy; +import org.eclipse.aether.repository.RemoteRepository; import org.eclipse.aether.resolution.DependencyRequest; import org.eclipse.aether.util.artifact.JavaScopes; import org.eclipse.aether.util.filter.DependencyFilterUtils; +import org.eclipse.aether.util.repository.AuthenticationBuilder; import java.io.File; import java.io.IOException; @@ -54,6 +61,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; @Command( @@ -196,6 +204,48 @@ public class PullDependencies implements Runnable ) public String defaultVersion = PullDependencies.class.getPackage().getImplementationVersion(); + @Option( + name = {"--use-proxy"}, + title = "Use http/https proxy to pull dependencies.", + required = false + ) + public boolean useProxy = false; + + @Option( + name = {"--proxy-type"}, + title = "The proxy type, should be either http or https", + required = false + ) + public String proxyType = "https"; + + @Option( + name = {"--proxy-host"}, + title = "The proxy host", + required = false + ) + public String proxyHost = ""; + + @Option( + name = {"--proxy-port"}, + title = "The proxy port", + required = false + ) + public int proxyPort = -1; + + @Option( + name = {"--proxy-username"}, + title = "The proxy username", + required = false + ) + public String proxyUsername = ""; + + @Option( + name = {"--proxy-password"}, + title = "The proxy password", + required = false + ) + public String proxyPassword = ""; + public PullDependencies() { } @@ -411,10 +461,7 @@ public class PullDependencies implements Runnable } if (log.isTraceEnabled() || log.isDebugEnabled()) { - return new DefaultTeslaAether( - localRepository, - remoteRepositories.toArray(new Repository[remoteRepositories.size()]) - ); + return createTeslaAether(remoteRepositories); } PrintStream oldOut = System.out; @@ -445,10 +492,7 @@ public class PullDependencies implements Runnable StringUtils.UTF8_STRING ) ); - return new DefaultTeslaAether( - localRepository, - remoteRepositories.toArray(new Repository[remoteRepositories.size()]) - ); + return createTeslaAether(remoteRepositories); } catch (UnsupportedEncodingException e) { // should never happen @@ -459,6 +503,39 @@ public class PullDependencies implements Runnable } } + private DefaultTeslaAether createTeslaAether(List remoteRepositories) + { + if (!useProxy) { + return new DefaultTeslaAether( + localRepository, + remoteRepositories.toArray(new Repository[remoteRepositories.size()]) + ); + } + + if (!StringUtils.toLowerCase(proxyType).equals(Proxy.TYPE_HTTP) && !StringUtils.toLowerCase(proxyType).equals(Proxy.TYPE_HTTPS)) { + throw new IllegalArgumentException("invalid proxy type: " + proxyType); + } + + RepositorySystemSession repositorySystemSession = new RepositorySystemSessionProvider(new File(localRepository)).get(); + List rl = remoteRepositories.stream().map(r -> { + RemoteRepository.Builder builder = new RemoteRepository.Builder(r.getId(), "default", r.getUrl()); + if (r.getUsername() != null && r.getPassword() != null) { + Authentication auth = new AuthenticationBuilder().addUsername(r.getUsername()).addPassword(r.getPassword()).build(); + builder.setAuthentication(auth); + } + + final Authentication proxyAuth; + if (Strings.isNullOrEmpty(proxyUsername)) { + proxyAuth = null; + } else { + proxyAuth = new AuthenticationBuilder().addUsername(proxyUsername).addPassword(proxyPassword).build(); + } + builder.setProxy(new Proxy(proxyType, proxyHost, proxyPort, proxyAuth)); + return builder.build(); + }).collect(Collectors.toList()); + return new DefaultTeslaAether(rl, repositorySystemSession); + } + /** * Create the extension directory for a specific maven coordinate. * The name of this directory should be the artifactId in the coordinate