HTTPCLIENT-2326: Propagate original proxy response to the caller in case of HTTP CONNECT request failure

httpclient5/src/main/java/org/apache/hc/client5/http/impl/TunnelRefusedException.java

@@ -28,25 +28,53 @@
 package org.apache.hc.client5.http.impl;
 
 import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.message.BasicHttpResponse;
+import org.apache.hc.core5.http.message.StatusLine;
+import org.conscrypt.Internal;
 
 /**
  * Signals that the tunnel request was rejected by the proxy host.
  *
  * @since 4.0
+ *
+ * @deprecated Do not use/
  */
+@Deprecated
 public class TunnelRefusedException extends HttpException {
 
     private static final long serialVersionUID = -8646722842745617323L;
 
-    private final String responseMessage;
+    private final HttpResponse response;
 
+    /**
+     * @deprecated Do not use.
+     */
+    @Deprecated
     public TunnelRefusedException(final String message, final String responseMessage) {
         super(message);
-        this.responseMessage = responseMessage;
+        this.response = new BasicHttpResponse(500);
     }
 
+    @Internal
+    public TunnelRefusedException(final HttpResponse response) {
+        super("CONNECT refused by proxy: " + new StatusLine(response));
+        this.response = response;
+    }
+
+    /**
+     * @deprecated Use {@link #getResponse()}.
+     */
+    @Deprecated
     public String getResponseMessage() {
-        return this.responseMessage;
+        return "CONNECT refused by proxy: " + new StatusLine(response);
+    }
+
+    /**
+     * @since 5.4
+     */
+    public HttpResponse getResponse() {
+        return response;
     }
 
 }

httpclient5/src/main/java/org/apache/hc/client5/http/impl/async/AsyncConnectExec.java

@@ -45,7 +45,6 @@
 import org.apache.hc.client5.http.auth.AuthExchange;
 import org.apache.hc.client5.http.auth.ChallengeType;
 import org.apache.hc.client5.http.config.RequestConfig;
-import org.apache.hc.client5.http.impl.TunnelRefusedException;
 import org.apache.hc.client5.http.impl.auth.AuthCacheKeeper;
 import org.apache.hc.client5.http.impl.auth.HttpAuthenticator;
 import org.apache.hc.client5.http.impl.routing.BasicRouteDirector;
@@ -121,8 +120,8 @@ static class State {
         final RouteTracker tracker;
 
         volatile boolean challenged;
+        volatile HttpResponse response;
         volatile boolean tunnelRefused;
-        volatile HttpResponse tunnelResponse;
 
     }
 
@@ -295,7 +294,7 @@ public void completed() {
                                         if (LOG.isDebugEnabled()) {
                                             LOG.debug("{} tunnel refused", exchangeId);
                                         }
-                                        asyncExecCallback.failed(new TunnelRefusedException("CONNECT refused by proxy: " + new StatusLine(state.tunnelResponse), null));
+                                        asyncExecCallback.completed();
                                     } else {
                                         if (LOG.isDebugEnabled()) {
                                             LOG.debug("{} tunnel to target created", exchangeId);
@@ -456,8 +455,7 @@ public void consumeResponse(final HttpResponse response,
                     state.challenged = false;
                     if (status >= HttpStatus.SC_REDIRECTION) {
                         state.tunnelRefused = true;
-                        state.tunnelResponse = response;
+                        entityConsumerRef.set(asyncExecCallback.handleResponse(response, entityDetails));
-                        entityConsumerRef.set(null);
                     } else if (status == HttpStatus.SC_OK) {
                         asyncExecCallback.completed();
                     } else {

httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/ConnectExec.java

@@ -40,7 +40,6 @@
 import org.apache.hc.client5.http.classic.ExecChainHandler;
 import org.apache.hc.client5.http.classic.ExecRuntime;
 import org.apache.hc.client5.http.config.RequestConfig;
-import org.apache.hc.client5.http.impl.TunnelRefusedException;
 import org.apache.hc.client5.http.impl.auth.AuthCacheKeeper;
 import org.apache.hc.client5.http.impl.auth.HttpAuthenticator;
 import org.apache.hc.client5.http.impl.routing.BasicRouteDirector;
@@ -52,6 +51,7 @@
 import org.apache.hc.core5.http.ClassicHttpRequest;
 import org.apache.hc.core5.http.ClassicHttpResponse;
 import org.apache.hc.core5.http.ConnectionReuseStrategy;
+import org.apache.hc.core5.http.ContentType;
 import org.apache.hc.core5.http.HttpEntity;
 import org.apache.hc.core5.http.HttpException;
 import org.apache.hc.core5.http.HttpHeaders;
@@ -60,6 +60,7 @@
 import org.apache.hc.core5.http.HttpStatus;
 import org.apache.hc.core5.http.HttpVersion;
 import org.apache.hc.core5.http.Method;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
 import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.apache.hc.core5.http.message.BasicClassicHttpRequest;
 import org.apache.hc.core5.http.message.StatusLine;
@@ -149,11 +150,15 @@ public ClassicHttpResponse execute(
                             tracker.connectProxy(proxy, route.isSecure() && !route.isTunnelled());
                             break;
                         case HttpRouteDirector.TUNNEL_TARGET: {
-                            final boolean secure = createTunnelToTarget(exchangeId, route, request, execRuntime, context);
+                            final ClassicHttpResponse finalResponse = createTunnelToTarget(
+                                    exchangeId, route, request, execRuntime, context);
+                            if (finalResponse != null) {
+                                return finalResponse;
+                            }
                             if (LOG.isDebugEnabled()) {
                                 LOG.debug("{} tunnel to target created.", exchangeId);
                             }
-                            tracker.tunnelTarget(secure);
+                            tracker.tunnelTarget(false);
                         }   break;
 
                         case HttpRouteDirector.TUNNEL_PROXY: {
@@ -207,7 +212,7 @@ public ClassicHttpResponse execute(
      * This method does <i>not</i> processChallenge the connection with
      * information about the tunnel, that is left to the caller.
      */
-    private boolean createTunnelToTarget(
+    private ClassicHttpResponse createTunnelToTarget(
             final String exchangeId,
             final HttpRoute route,
             final HttpRequest request,
@@ -282,16 +287,16 @@ private boolean createTunnelToTarget(
 
         final int status = response.getCode();
         if (status != HttpStatus.SC_OK) {
-            EntityUtils.consume(response.getEntity());
+            final HttpEntity entity = response.getEntity();
-            execRuntime.disconnectEndpoint();
+            if (entity != null) {
-            throw new TunnelRefusedException("CONNECT refused by proxy: " + new StatusLine(response), null);
+                response.setEntity(new ByteArrayEntity(
+                        EntityUtils.toByteArray(entity, 4096),
+                        ContentType.parseLenient(entity.getContentType())));
+                execRuntime.disconnectEndpoint();
+            }
+            return response;
         }
-
+        return null;
-        // How to decide on security of the tunnelled connection?
-        // The socket factory knows only about the segment to the proxy.
-        // Even if that is secure, the hop to the target may be insecure.
-        // Leave it to derived classes, consider insecure by default here.
-        return false;
     }
 
     /**

httpclient5/src/main/java/org/apache/hc/client5/http/impl/classic/ProxyClient.java

@@ -43,7 +43,6 @@
 import org.apache.hc.client5.http.config.RequestConfig;
 import org.apache.hc.client5.http.impl.DefaultAuthenticationStrategy;
 import org.apache.hc.client5.http.impl.DefaultClientConnectionReuseStrategy;
-import org.apache.hc.client5.http.impl.TunnelRefusedException;
 import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
 import org.apache.hc.client5.http.impl.auth.BasicSchemeFactory;
 import org.apache.hc.client5.http.impl.auth.DigestSchemeFactory;
@@ -202,7 +201,7 @@ public Socket tunnel(
         if (status > 299) {
             EntityUtils.consume(response.getEntity());
             conn.close();
-            throw new TunnelRefusedException("CONNECT refused by proxy: " + new StatusLine(response), null);
+            throw new HttpException("Tunnel refused: " + new StatusLine(response));
         }
         return conn.getSocket();
     }

httpclient5/src/test/java/org/apache/hc/client5/http/impl/classic/TestConnectExec.java

@@ -41,7 +41,6 @@
 import org.apache.hc.client5.http.classic.ExecRuntime;
 import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.apache.hc.client5.http.entity.EntityBuilder;
-import org.apache.hc.client5.http.impl.TunnelRefusedException;
 import org.apache.hc.client5.http.impl.auth.BasicScheme;
 import org.apache.hc.client5.http.impl.auth.CredentialsProviderBuilder;
 import org.apache.hc.client5.http.protocol.HttpClientContext;
@@ -218,9 +217,8 @@ public void testEstablishRouteViaProxyTunnelFailure() throws Exception {
                 Mockito.any())).thenReturn(response);
 
         final ExecChain.Scope scope = new ExecChain.Scope("test", route, request, execRuntime, context);
-        Assertions.assertThrows(TunnelRefusedException.class, () -> exec.execute(request, scope, execChain));
+        exec.execute(request, scope, execChain);
         Mockito.verify(execRuntime, Mockito.atLeastOnce()).disconnectEndpoint();
-        Mockito.verify(execRuntime).discardEndpoint();
     }
 
     @Test