Add http/https proxy options to PullDependencies.java (#5450)

docs/content/operations/pull-deps.md

@@ -40,6 +40,30 @@ Don't use the default remote repositories, only use the repositories provided di
 
 Version to use for extension coordinate that doesn't have a version information. For example, if extension coordinate is `io.druid.extensions:mysql-metadata-storage`, and default version is `#{DRUIDVERSION}`, then this coordinate will be treated as `io.druid.extensions:mysql-metadata-storage:#{DRUIDVERSION}`
 
+`--use-proxy`
+
+Use http/https proxy to send request to the remote repository servers. `--proxy-host` and `--proxy-port` must be set explicitly if this option is enabled.  
+
+`--proxy-type`
+
+Set the proxy type, Should be either *http* or *https*, default value is *https*.
+
+`--proxy-host`
+
+Set the proxy host. e.g. proxy.com.
+
+`--proxy-port`
+
+Set the proxy port number. e.g. 8080.
+
+`--proxy-username`
+
+Set a username to connect to the proxy, this option is only required if the proxy server uses authentication.
+
+`--proxy-password`
+
+Set a password to connect to the proxy, this option is only required if the proxy server uses authentication.
+
 To run `pull-deps`, you should
 
 1) Specify `druid.extensions.directory` and `druid.extensions.hadoopDependenciesDir`, these two properties tell `pull-deps` where to put extensions. If you don't specify them,  default values will be used, see [Configuration](../configuration/index.html).

services/src/main/java/io/druid/cli/PullDependencies.java

@@ -19,6 +19,7 @@
 
 package io.druid.cli;
 
+import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -33,17 +34,23 @@ import io.druid.java.util.common.StringUtils;
 import io.druid.java.util.common.logger.Logger;
 import io.tesla.aether.Repository;
 import io.tesla.aether.TeslaAether;
+import io.tesla.aether.guice.RepositorySystemSessionProvider;
 import io.tesla.aether.internal.DefaultTeslaAether;
 import org.apache.commons.io.FileUtils;
+import org.eclipse.aether.RepositorySystemSession;
 import org.eclipse.aether.artifact.Artifact;
 import org.eclipse.aether.artifact.DefaultArtifact;
 import org.eclipse.aether.collection.CollectRequest;
 import org.eclipse.aether.graph.Dependency;
 import org.eclipse.aether.graph.DependencyFilter;
 import org.eclipse.aether.graph.DependencyNode;
+import org.eclipse.aether.repository.Authentication;
+import org.eclipse.aether.repository.Proxy;
+import org.eclipse.aether.repository.RemoteRepository;
 import org.eclipse.aether.resolution.DependencyRequest;
 import org.eclipse.aether.util.artifact.JavaScopes;
 import org.eclipse.aether.util.filter.DependencyFilterUtils;
+import org.eclipse.aether.util.repository.AuthenticationBuilder;
 
 import java.io.File;
 import java.io.IOException;
@@ -54,6 +61,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 
 @Command(
@@ -196,6 +204,48 @@ public class PullDependencies implements Runnable
   )
   public String defaultVersion = PullDependencies.class.getPackage().getImplementationVersion();
 
+  @Option(
+      name = {"--use-proxy"},
+      title = "Use http/https proxy to pull dependencies.",
+      required = false
+  )
+  public boolean useProxy = false;
+
+  @Option(
+      name = {"--proxy-type"},
+      title = "The proxy type, should be either http or https",
+      required = false
+  )
+  public String proxyType = "https";
+
+  @Option(
+      name = {"--proxy-host"},
+      title = "The proxy host",
+      required = false
+  )
+  public String proxyHost = "";
+
+  @Option(
+      name = {"--proxy-port"},
+      title = "The proxy port",
+      required = false
+  )
+  public int proxyPort = -1;
+
+  @Option(
+      name = {"--proxy-username"},
+      title = "The proxy username",
+      required = false
+  )
+  public String proxyUsername = "";
+
+  @Option(
+      name = {"--proxy-password"},
+      title = "The proxy password",
+      required = false
+  )
+  public String proxyPassword = "";
+
   public PullDependencies()
   {
   }
@@ -411,10 +461,7 @@ public class PullDependencies implements Runnable
     }
 
     if (log.isTraceEnabled() || log.isDebugEnabled()) {
-      return new DefaultTeslaAether(
+      return createTeslaAether(remoteRepositories);
-          localRepository,
-          remoteRepositories.toArray(new Repository[remoteRepositories.size()])
-      );
     }
 
     PrintStream oldOut = System.out;
@@ -445,10 +492,7 @@ public class PullDependencies implements Runnable
               StringUtils.UTF8_STRING
           )
       );
-      return new DefaultTeslaAether(
+      return createTeslaAether(remoteRepositories);
-          localRepository,
-          remoteRepositories.toArray(new Repository[remoteRepositories.size()])
-      );
     }
     catch (UnsupportedEncodingException e) {
       // should never happen
@@ -459,6 +503,39 @@ public class PullDependencies implements Runnable
     }
   }
 
+  private DefaultTeslaAether createTeslaAether(List<Repository> remoteRepositories)
+  {
+    if (!useProxy) {
+      return new DefaultTeslaAether(
+          localRepository,
+          remoteRepositories.toArray(new Repository[remoteRepositories.size()])
+      );
+    }
+
+    if (!StringUtils.toLowerCase(proxyType).equals(Proxy.TYPE_HTTP) && !StringUtils.toLowerCase(proxyType).equals(Proxy.TYPE_HTTPS)) {
+      throw new IllegalArgumentException("invalid proxy type: " + proxyType);
+    }
+
+    RepositorySystemSession repositorySystemSession = new RepositorySystemSessionProvider(new File(localRepository)).get();
+    List<RemoteRepository> rl = remoteRepositories.stream().map(r -> {
+      RemoteRepository.Builder builder = new RemoteRepository.Builder(r.getId(), "default", r.getUrl());
+      if (r.getUsername() != null && r.getPassword() != null) {
+        Authentication auth = new AuthenticationBuilder().addUsername(r.getUsername()).addPassword(r.getPassword()).build();
+        builder.setAuthentication(auth);
+      }
+
+      final Authentication proxyAuth;
+      if (Strings.isNullOrEmpty(proxyUsername)) {
+        proxyAuth = null;
+      } else {
+        proxyAuth = new AuthenticationBuilder().addUsername(proxyUsername).addPassword(proxyPassword).build();
+      }
+      builder.setProxy(new Proxy(proxyType, proxyHost, proxyPort, proxyAuth));
+      return builder.build();
+    }).collect(Collectors.toList());
+    return new DefaultTeslaAether(rl, repositorySystemSession);
+  }
+
   /**
    * Create the extension directory for a specific maven coordinate.
    * The name of this directory should be the artifactId in the coordinate