Update httpclient for JDK 11 TLS engine (#37994)

The apache commons http client implementations recently released
versions that solve TLS compatibility issues with the new TLS engine
that supports TLSv1.3 with JDK 11. This change updates our code to
use these versions since JDK 11 is a supported JDK and we should
allow the use of TLSv1.3.

buildSrc/version.properties

@@ -21,16 +21,13 @@ joda              = 2.10.1
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.12
-httpclient        = 4.5.2
+httpclient        = 4.5.7
-# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+httpcore          = 4.4.11
-httpcore          = 4.4.5
+httpasyncclient   = 4.1.4
-# When updating httpasyncclient, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpasyncclient   = 4.1.2
 commonslogging    = 1.1.3
-commonscodec      = 1.10
+commonscodec      = 1.11
 hamcrest          = 1.3
 securemock        = 1.2
-# When updating mocksocket, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 mocksocket        = 1.2
 
 # benchmark dependencies

client/rest/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

client/rest/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

client/rest/licenses/httpasyncclient-4.1.2.jar.sha1

@@ -1 +0,0 @@
-95aa3e6fb520191a0970a73cf09f62948ee614be

client/rest/licenses/httpasyncclient-4.1.4.jar.sha1

@@ -0,0 +1 @@
+f3a3240681faae3fa46b573a4c7e50cec9db0d86

client/rest/licenses/httpclient-4.5.2.jar.sha1

@@ -1 +0,0 @@
-733db77aa8d9b2d68015189df76ab06304406e50

client/rest/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

client/rest/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

client/rest/licenses/httpcore-4.4.5.jar.sha1

@@ -1 +0,0 @@
-e7501a1b34325abb00d17dde96150604a0658b54

client/rest/licenses/httpcore-nio-4.4.11.jar.sha1

@@ -0,0 +1 @@
+7d0a97d01d39cff9aa3e6db81f21fddb2435f4e6

client/rest/licenses/httpcore-nio-4.4.5.jar.sha1

@@ -1 +0,0 @@
-f4be009e7505f6ceddf21e7960c759f413f15056

client/sniffer/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

client/sniffer/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

client/sniffer/licenses/httpclient-4.5.2.jar.sha1

@@ -1 +0,0 @@
-733db77aa8d9b2d68015189df76ab06304406e50

client/sniffer/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

client/sniffer/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

client/sniffer/licenses/httpcore-4.4.5.jar.sha1

@@ -1 +0,0 @@
-e7501a1b34325abb00d17dde96150604a0658b54

plugins/analysis-phonetic/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

plugins/analysis-phonetic/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

plugins/discovery-azure-classic/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1

@@ -1 +0,0 @@
-733db77aa8d9b2d68015189df76ab06304406e50

plugins/discovery-azure-classic/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/discovery-azure-classic/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/discovery-azure-classic/licenses/httpcore-4.4.5.jar.sha1

@@ -1 +0,0 @@
-e7501a1b34325abb00d17dde96150604a0658b54

plugins/discovery-ec2/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

plugins/discovery-ec2/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1

@@ -1 +0,0 @@
-733db77aa8d9b2d68015189df76ab06304406e50

plugins/discovery-ec2/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/discovery-ec2/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/discovery-ec2/licenses/httpcore-4.4.5.jar.sha1

@@ -1 +0,0 @@
-e7501a1b34325abb00d17dde96150604a0658b54

plugins/discovery-gce/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

plugins/discovery-gce/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1

@@ -1 +0,0 @@
-733db77aa8d9b2d68015189df76ab06304406e50

plugins/discovery-gce/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/discovery-gce/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/discovery-gce/licenses/httpcore-4.4.5.jar.sha1

@@ -1 +0,0 @@
-e7501a1b34325abb00d17dde96150604a0658b54

plugins/ingest-attachment/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

plugins/ingest-attachment/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

plugins/repository-gcs/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1

@@ -1 +0,0 @@
-733db77aa8d9b2d68015189df76ab06304406e50

plugins/repository-gcs/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/repository-gcs/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/repository-gcs/licenses/httpcore-4.4.5.jar.sha1

@@ -1 +0,0 @@
-e7501a1b34325abb00d17dde96150604a0658b54

plugins/repository-hdfs/build.gradle

@@ -52,7 +52,7 @@ dependencies {
   compile 'com.google.protobuf:protobuf-java:2.5.0'
   compile 'commons-logging:commons-logging:1.1.3'
   compile 'commons-cli:commons-cli:1.2'
-  compile 'commons-codec:commons-codec:1.10'
+  compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile 'commons-collections:commons-collections:3.2.2'
   compile 'commons-configuration:commons-configuration:1.6'
   compile 'commons-io:commons-io:2.4'

plugins/repository-hdfs/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

plugins/repository-hdfs/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/repository-s3/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

plugins/repository-s3/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1

@@ -1 +0,0 @@
-733db77aa8d9b2d68015189df76ab06304406e50

plugins/repository-s3/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

plugins/repository-s3/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

plugins/repository-s3/licenses/httpcore-4.4.5.jar.sha1

@@ -1 +0,0 @@
-e7501a1b34325abb00d17dde96150604a0658b54

x-pack/plugin/core/licenses/commons-codec-1.10.jar.sha1

@@ -1 +0,0 @@
-4b95f4897fa13f2cd904aee711aeafc0c5295cd8

x-pack/plugin/core/licenses/commons-codec-1.11.jar.sha1

@@ -0,0 +1 @@
+3acb4705652e16236558f0f4f2192cc33c3bd189

x-pack/plugin/core/licenses/httpasyncclient-4.1.2.jar.sha1

@@ -1 +0,0 @@
-95aa3e6fb520191a0970a73cf09f62948ee614be

x-pack/plugin/core/licenses/httpasyncclient-4.1.4.jar.sha1

@@ -0,0 +1 @@
+f3a3240681faae3fa46b573a4c7e50cec9db0d86

x-pack/plugin/core/licenses/httpclient-4.5.2.jar.sha1

@@ -1 +0,0 @@
-733db77aa8d9b2d68015189df76ab06304406e50

x-pack/plugin/core/licenses/httpclient-4.5.7.jar.sha1

@@ -0,0 +1 @@
+dda059f4908e1b548b7ba68d81a3b05897f27cb0

x-pack/plugin/core/licenses/httpcore-4.4.11.jar.sha1

@@ -0,0 +1 @@
+de748cf874e4e193b42eceea9fe5574fabb9d4df

x-pack/plugin/core/licenses/httpcore-4.4.5.jar.sha1

@@ -1 +0,0 @@
-e7501a1b34325abb00d17dde96150604a0658b54

x-pack/plugin/core/licenses/httpcore-nio-4.4.11.jar.sha1

@@ -0,0 +1 @@
+7d0a97d01d39cff9aa3e6db81f21fddb2435f4e6

x-pack/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1

@@ -1 +0,0 @@
-f4be009e7505f6ceddf21e7960c759f413f15056

x-pack/plugin/security/licenses/httpclient-cache-4.5.2.jar.sha1

@@ -1 +0,0 @@
-bd50ea83908dbf2f387a333216e66d2f0c5079bd

x-pack/plugin/security/licenses/httpclient-cache-4.5.7.jar.sha1

@@ -0,0 +1 @@
+c13a0ce27c17831e5e5be6c751842006dcecb270

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java

@@ -47,6 +47,7 @@ import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -162,7 +163,9 @@ public class HttpClient implements Closeable {
     }
 
     public HttpResponse execute(HttpRequest request) throws IOException {
-        URI uri = createURI(request);
+        Tuple<HttpHost, URI> tuple = createURI(request);
+        final URI uri = tuple.v2();
+        final HttpHost httpHost = tuple.v1();
 
         HttpRequestBase internalRequest;
         if (request.method == HttpMethod.HEAD) {
@@ -212,7 +215,7 @@ public class HttpClient implements Closeable {
             // preemptive auth, no need to wait for a 401 first
             AuthCache authCache = new BasicAuthCache();
             BasicScheme basicAuth = new BasicScheme();
-            authCache.put(new HttpHost(request.host, request.port, request.scheme.scheme()), basicAuth);
+            authCache.put(httpHost, basicAuth);
             localContext.setAuthCache(authCache);
         }
 
@@ -233,7 +236,7 @@ public class HttpClient implements Closeable {
 
         internalRequest.setConfig(config.build());
 
-        try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(internalRequest, localContext))) {
+        try (CloseableHttpResponse response = SocketAccess.doPrivileged(() -> client.execute(httpHost, internalRequest, localContext))) {
             // headers
             Header[] headers = response.getAllHeaders();
             Map<String, String[]> responseHeaders = new HashMap<>(headers.length);
@@ -310,7 +313,7 @@ public class HttpClient implements Closeable {
         return HttpProxy.NO_PROXY;
     }
 
-    private URI createURI(HttpRequest request) {
+    private Tuple<HttpHost, URI> createURI(HttpRequest request) {
         // this could be really simple, as the apache http client has a UriBuilder class, however this class is always doing
         // url path escaping, and we have done this already, so this would result in double escaping
         try {
@@ -320,7 +323,23 @@ public class HttpClient implements Closeable {
             URI uri = URIUtils.createURI(request.scheme.scheme(), request.host, request.port, request.path,
                     Strings.isNullOrEmpty(format) ? null : format, null);
 
-            return uri;
+            if (uri.isAbsolute() == false) {
+                throw new IllegalStateException("URI [" + uri.toASCIIString() + "] must be absolute");
+            }
+            final HttpHost httpHost = URIUtils.extractHost(uri);
+            // what a mess that we need to do this to workaround https://issues.apache.org/jira/browse/HTTPCLIENT-1968
+            // in some cases the HttpClient will re-write the URI which drops the escaping for
+            // slashes within a path. This rewriting is done to obtain a relative URI when
+            // a proxy is not being used. To avoid this we can handle making it relative ourselves
+            if (request.path != null && request.path.contains("%2F")) {
+                final boolean isUsingProxy = (request.proxy != null && request.proxy.equals(HttpProxy.NO_PROXY) == false) ||
+                    HttpProxy.NO_PROXY.equals(settingsProxy) == false;
+                if (isUsingProxy == false) {
+                    // we need a relative uri
+                    uri = URIUtils.createURI(null, null, -1, request.path, Strings.isNullOrEmpty(format) ? null : format, null);
+                }
+            }
+            return new Tuple<>(httpHost, uri);
         } catch (URISyntaxException e) {
             throw new IllegalArgumentException(e);
         }