NETWORKING: Fix Netty Leaks by upgrading to 4.1.28 (#32511)

* Upgrade to `4.1.28` since the problem reported in #32487 is a bug in Netty itself (see https://github.com/netty/netty/issues/7337)
* Fixed other leaks in test code that now showed up due to fixes improvements in leak reporting in the newer version
* Needed to extend permissions for netty common package because it now sets a classloader at runtime after changes in 63bae0956a
* Adjusted forbidden APIs check accordingly
* Closes #32487

modules/transport-netty4/build.gradle

@@ -34,13 +34,13 @@ compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-tr
 
 dependencies {
     // network stack
-    compile "io.netty:netty-buffer:4.1.16.Final"
+    compile "io.netty:netty-buffer:4.1.28.Final"
-    compile "io.netty:netty-codec:4.1.16.Final"
+    compile "io.netty:netty-codec:4.1.28.Final"
-    compile "io.netty:netty-codec-http:4.1.16.Final"
+    compile "io.netty:netty-codec-http:4.1.28.Final"
-    compile "io.netty:netty-common:4.1.16.Final"
+    compile "io.netty:netty-common:4.1.28.Final"
-    compile "io.netty:netty-handler:4.1.16.Final"
+    compile "io.netty:netty-handler:4.1.28.Final"
-    compile "io.netty:netty-resolver:4.1.16.Final"
+    compile "io.netty:netty-resolver:4.1.28.Final"
-    compile "io.netty:netty-transport:4.1.16.Final"
+    compile "io.netty:netty-transport:4.1.28.Final"
 }
 
 dependencyLicenses {
@@ -134,7 +134,6 @@ thirdPartyAudit.excludes = [
         'net.jpountz.xxhash.StreamingXXHash32',
         'net.jpountz.xxhash.XXHashFactory',
         'io.netty.internal.tcnative.CertificateRequestedCallback',
-        'io.netty.internal.tcnative.CertificateRequestedCallback$KeyMaterial',
         'io.netty.internal.tcnative.CertificateVerifier',
         'io.netty.internal.tcnative.SessionTicketKey',
         'io.netty.internal.tcnative.SniHostNameMatcher',
@@ -161,6 +160,6 @@ thirdPartyAudit.excludes = [
 
         'org.conscrypt.AllocatedBuffer',
         'org.conscrypt.BufferAllocator',
-        'org.conscrypt.Conscrypt$Engines',
+        'org.conscrypt.Conscrypt',
         'org.conscrypt.HandshakeListener'
 ]

modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-63b5fa95c74785e16f2c30ce268bc222e35c8cb5

modules/transport-netty4/licenses/netty-buffer-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+d6c2d13492778009d33f60e05ed90bcb535d1fd1

modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-d84a1f21768b7309c2954521cf5a1f46c2309eb1

modules/transport-netty4/licenses/netty-codec-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+a38361d893900947524f8a9da980555950e73d6a

modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-d64312378b438dfdad84267c599a053327c6f02a

modules/transport-netty4/licenses/netty-codec-http-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+897100c1022c780b0a436b9349e507e8fa9800dc

modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-177a6b30cca92f6f5f9873c9befd681377a4c328

modules/transport-netty4/licenses/netty-common-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+df69ce8bb9b544a71e7bbee290253cf7c93e6bad

modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-fec0e63e7dd7f4eeef7ea8dc47a1ff32dfc7ebc2

modules/transport-netty4/licenses/netty-handler-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+a035784682da0126bc25f10713dac732b5082a6d

modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-f6eb553b53fb3a90a8ac1170697093fed82eae28

modules/transport-netty4/licenses/netty-resolver-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+f33557dcb31fa20da075ac05e4808115e32ef9b7

modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-3c8ee2c4d4a1cbb947a5c184c7aeb2204260958b

modules/transport-netty4/licenses/netty-transport-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+d2ef28f49d726737f0ffe84bf66529b3bf6e0c0d

modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy

@@ -23,6 +23,9 @@ grant codeBase "${codebase.netty-common}" {
 
    // netty makes and accepts socket connections
    permission java.net.SocketPermission "*", "accept,connect";
+
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -92,15 +93,19 @@ public class Netty4BadRequestTests extends ESTestCase {
             try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
                 final Collection<FullHttpResponse> responses =
                         nettyHttpClient.get(transportAddress.address(), "/_cluster/settings?pretty=%");
-                assertThat(responses, hasSize(1));
+                try {
-                assertThat(responses.iterator().next().status().code(), equalTo(400));
+                    assertThat(responses, hasSize(1));
-                final Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
+                    assertThat(responses.iterator().next().status().code(), equalTo(400));
-                assertThat(responseBodies, hasSize(1));
+                    final Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
-                assertThat(responseBodies.iterator().next(), containsString("\"type\":\"bad_parameter_exception\""));
+                    assertThat(responseBodies, hasSize(1));
-                assertThat(
+                    assertThat(responseBodies.iterator().next(), containsString("\"type\":\"bad_parameter_exception\""));
+                    assertThat(
                         responseBodies.iterator().next(),
                         containsString(
-                                "\"reason\":\"java.lang.IllegalArgumentException: unterminated escape sequence at end of string: %\""));
+                            "\"reason\":\"java.lang.IllegalArgumentException: unterminated escape sequence at end of string: %\""));
+                } finally {
+                    responses.forEach(ReferenceCounted::release);
+                }
             }
         }
     }

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ESNetty4IntegTestCase;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
@@ -88,12 +89,20 @@ public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase {
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> singleResponse = nettyHttpClient.post(transportAddress.address(), requests[0]);
-            assertThat(singleResponse, hasSize(1));
+            try {
-            assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK);
+                assertThat(singleResponse, hasSize(1));
+                assertAtLeastOnceExpectedStatus(singleResponse, HttpResponseStatus.OK);
 
-            Collection<FullHttpResponse> multipleResponses = nettyHttpClient.post(transportAddress.address(), requests);
+                Collection<FullHttpResponse> multipleResponses = nettyHttpClient.post(transportAddress.address(), requests);
-            assertThat(multipleResponses, hasSize(requests.length));
+                try {
-            assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE);
+                    assertThat(multipleResponses, hasSize(requests.length));
+                    assertAtLeastOnceExpectedStatus(multipleResponses, HttpResponseStatus.SERVICE_UNAVAILABLE);
+                } finally {
+                    multipleResponses.forEach(ReferenceCounted::release);
+                }
+            } finally {
+                singleResponse.forEach(ReferenceCounted::release);
+            }
         }
     }
 
@@ -113,8 +122,12 @@ public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase {
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> responses = nettyHttpClient.put(transportAddress.address(), requestUris);
-            assertThat(responses, hasSize(requestUris.length));
+            try {
-            assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
+                assertThat(responses, hasSize(requestUris.length));
+                assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
         }
     }
 

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java

@@ -29,6 +29,7 @@ import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -98,8 +99,12 @@ public class Netty4HttpServerPipeliningTests extends ESTestCase {
 
             try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
                 Collection<FullHttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests.toArray(new String[]{}));
-                Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
+                try {
-                assertThat(responseBodies, contains(requests.toArray()));
+                    Collection<String> responseBodies = Netty4HttpClient.returnHttpResponseBodies(responses);
+                    assertThat(responseBodies, contains(requests.toArray()));
+                } finally {
+                    responses.forEach(ReferenceCounted::release);
+                }
             }
         }
     }

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java

@@ -207,14 +207,23 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
                 HttpUtil.setContentLength(request, contentLength);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(expectedStatus));
+                try {
-                if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
+                    assertThat(response.status(), equalTo(expectedStatus));
-                    final FullHttpRequest continuationRequest =
+                    if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
-                        new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
+                        final FullHttpRequest continuationRequest =
-                    final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
+                            new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
-
+                        final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
-                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                        try {
-                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                            assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                            assertThat(
+                                new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done")
+                            );
+                        } finally {
+                            continuationResponse.release();
+                        }
+                    }
+                } finally {
+                    response.release();
                 }
             }
         }
@@ -280,10 +289,14 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
                 final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
+                try {
-                assertThat(
+                    assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
+                    assertThat(
                         new String(response.content().array(), Charset.forName("UTF-8")),
                         containsString("you sent a bad request and you should feel bad"));
+                } finally {
+                    response.release();
+                }
             }
         }
 

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.http.netty4;
 
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.util.ReferenceCounted;
 import org.elasticsearch.ESNetty4IntegTestCase;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
@@ -45,14 +46,18 @@ public class Netty4PipeliningIT extends ESNetty4IntegTestCase {
 
         HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
         TransportAddress[] boundAddresses = httpServerTransport.boundAddress().boundAddresses();
-        TransportAddress transportAddress = (TransportAddress) randomFrom(boundAddresses);
+        TransportAddress transportAddress = randomFrom(boundAddresses);
 
         try (Netty4HttpClient nettyHttpClient = new Netty4HttpClient()) {
             Collection<FullHttpResponse> responses = nettyHttpClient.get(transportAddress.address(), requests);
-            assertThat(responses, hasSize(5));
+            try {
+                assertThat(responses, hasSize(5));
 
-            Collection<String> opaqueIds = Netty4HttpClient.returnOpaqueIds(responses);
+                Collection<String> opaqueIds = Netty4HttpClient.returnOpaqueIds(responses);
-            assertOpaqueIdsInOrder(opaqueIds);
+                assertOpaqueIdsInOrder(opaqueIds);
+            } finally {
+                responses.forEach(ReferenceCounted::release);
+            }
         }
     }
 

plugins/transport-nio/build.gradle

@@ -29,13 +29,13 @@ dependencies {
     compile "org.elasticsearch:elasticsearch-nio:${version}"
 
     // network stack
-    compile "io.netty:netty-buffer:4.1.16.Final"
+    compile "io.netty:netty-buffer:4.1.28.Final"
-    compile "io.netty:netty-codec:4.1.16.Final"
+    compile "io.netty:netty-codec:4.1.28.Final"
-    compile "io.netty:netty-codec-http:4.1.16.Final"
+    compile "io.netty:netty-codec-http:4.1.28.Final"
-    compile "io.netty:netty-common:4.1.16.Final"
+    compile "io.netty:netty-common:4.1.28.Final"
-    compile "io.netty:netty-handler:4.1.16.Final"
+    compile "io.netty:netty-handler:4.1.28.Final"
-    compile "io.netty:netty-resolver:4.1.16.Final"
+    compile "io.netty:netty-resolver:4.1.28.Final"
-    compile "io.netty:netty-transport:4.1.16.Final"
+    compile "io.netty:netty-transport:4.1.28.Final"
 }
 
 dependencyLicenses {
@@ -113,7 +113,6 @@ thirdPartyAudit.excludes = [
         'net.jpountz.xxhash.StreamingXXHash32',
         'net.jpountz.xxhash.XXHashFactory',
         'io.netty.internal.tcnative.CertificateRequestedCallback',
-        'io.netty.internal.tcnative.CertificateRequestedCallback$KeyMaterial',
         'io.netty.internal.tcnative.CertificateVerifier',
         'io.netty.internal.tcnative.SessionTicketKey',
         'io.netty.internal.tcnative.SniHostNameMatcher',
@@ -140,6 +139,6 @@ thirdPartyAudit.excludes = [
 
         'org.conscrypt.AllocatedBuffer',
         'org.conscrypt.BufferAllocator',
-        'org.conscrypt.Conscrypt$Engines',
+        'org.conscrypt.Conscrypt',
         'org.conscrypt.HandshakeListener'
 ]

plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-63b5fa95c74785e16f2c30ce268bc222e35c8cb5

plugins/transport-nio/licenses/netty-buffer-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+d6c2d13492778009d33f60e05ed90bcb535d1fd1

plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-d84a1f21768b7309c2954521cf5a1f46c2309eb1

plugins/transport-nio/licenses/netty-codec-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+a38361d893900947524f8a9da980555950e73d6a

plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-d64312378b438dfdad84267c599a053327c6f02a

plugins/transport-nio/licenses/netty-codec-http-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+897100c1022c780b0a436b9349e507e8fa9800dc

plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-177a6b30cca92f6f5f9873c9befd681377a4c328

plugins/transport-nio/licenses/netty-common-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+df69ce8bb9b544a71e7bbee290253cf7c93e6bad

plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-fec0e63e7dd7f4eeef7ea8dc47a1ff32dfc7ebc2

plugins/transport-nio/licenses/netty-handler-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+a035784682da0126bc25f10713dac732b5082a6d

plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-f6eb553b53fb3a90a8ac1170697093fed82eae28

plugins/transport-nio/licenses/netty-resolver-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+f33557dcb31fa20da075ac05e4808115e32ef9b7

plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1

@@ -1 +0,0 @@
-3c8ee2c4d4a1cbb947a5c184c7aeb2204260958b

plugins/transport-nio/licenses/netty-transport-4.1.28.Final.jar.sha1

@@ -0,0 +1 @@
+d2ef28f49d726737f0ffe84bf66529b3bf6e0c0d

plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy

@@ -26,4 +26,6 @@ grant codeBase "${codebase.netty-common}" {
    // This should only currently be required as we use the netty http client for tests
    // netty makes and accepts socket connections
    permission java.net.SocketPermission "*", "accept,connect";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };

plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java

@@ -198,14 +198,23 @@ public class NioHttpServerTransportTests extends ESTestCase {
                 HttpUtil.setContentLength(request, contentLength);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(expectedStatus));
+                try {
-                if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
+                    assertThat(response.status(), equalTo(expectedStatus));
-                    final FullHttpRequest continuationRequest =
+                    if (expectedStatus.equals(HttpResponseStatus.CONTINUE)) {
-                        new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
+                        final FullHttpRequest continuationRequest =
-                    final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
+                            new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/", Unpooled.EMPTY_BUFFER);
-
+                        final FullHttpResponse continuationResponse = client.post(remoteAddress.address(), continuationRequest);
-                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                        try {
-                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                            assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                            assertThat(
+                                new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done")
+                            );
+                        } finally {
+                            continuationResponse.release();
+                        }
+                    }
+                } finally {
+                    response.release();
                 }
             }
         }
@@ -271,10 +280,14 @@ public class NioHttpServerTransportTests extends ESTestCase {
                 final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
 
                 final FullHttpResponse response = client.post(remoteAddress.address(), request);
-                assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
+                try {
-                assertThat(
+                    assertThat(response.status(), equalTo(HttpResponseStatus.BAD_REQUEST));
-                    new String(response.content().array(), Charset.forName("UTF-8")),
+                    assertThat(
-                    containsString("you sent a bad request and you should feel bad"));
+                        new String(response.content().array(), Charset.forName("UTF-8")),
+                        containsString("you sent a bad request and you should feel bad"));
+                } finally {
+                    response.release();
+                }
             }
         }
 

x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy

@@ -15,6 +15,8 @@ grant {
 grant codeBase "${codebase.netty-common}" {
    // for reading the system-wide configuration for the backlog of established sockets
    permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {

x-pack/plugin/ml/src/main/plugin-metadata/plugin-security.policy

@@ -9,6 +9,8 @@ grant {
 grant codeBase "${codebase.netty-common}" {
    // for reading the system-wide configuration for the backlog of established sockets
    permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
+   // Netty sets custom classloader for some of its internal threads
+   permission java.lang.RuntimePermission "*", "setContextClassLoader";
 };
 
 grant codeBase "${codebase.netty-transport}" {