Upgrade Netty to 4.1.x (#6417)

* Update netty to 4.1.30.Final

* Fix compile time problems with new netty

* Remove netty-all from rocketmq extension

extensions-contrib/druid-rocketmq/pom.xml

@@ -40,6 +40,13 @@
       <groupId>com.alibaba.rocketmq</groupId>
       <artifactId>rocketmq-client</artifactId>
       <version>${rocketmq.version}</version>
+      <exclusions>
+        <!-- Druid uses its own netty version -->
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty-all</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.druid</groupId>

extensions-core/druid-basic-security/src/test/java/org/apache/druid/security/authentication/BasicHTTPAuthenticatorTest.java

@@ -31,7 +31,6 @@ import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorC
 import org.apache.druid.security.basic.authentication.entity.BasicAuthenticatorUser;
 import org.apache.druid.server.security.AuthConfig;
 import org.apache.druid.server.security.AuthenticationResult;
-import org.asynchttpclient.util.Base64;
 import org.easymock.EasyMock;
 import org.junit.Test;
 
@@ -82,9 +81,7 @@ public class BasicHTTPAuthenticatorTest
   @Test
   public void testGoodPassword() throws IOException, ServletException
   {
-    String header = Base64.encode(
+    String header = StringUtils.utf8Base64("userA:helloworld");
-        StringUtils.toUtf8("userA:helloworld")
-    );
     header = StringUtils.format("Basic %s", header);
 
     HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class);
@@ -113,9 +110,7 @@ public class BasicHTTPAuthenticatorTest
   @Test
   public void testBadPassword() throws IOException, ServletException
   {
-    String header = Base64.encode(
+    String header = StringUtils.utf8Base64("userA:badpassword");
-        StringUtils.toUtf8("userA:badpassword")
-    );
     header = StringUtils.format("Basic %s", header);
 
     HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class);
@@ -139,9 +134,7 @@ public class BasicHTTPAuthenticatorTest
   @Test
   public void testUnknownUser() throws IOException, ServletException
   {
-    String header = Base64.encode(
+    String header = StringUtils.utf8Base64("userB:helloworld");
-        StringUtils.toUtf8("userB:helloworld")
-    );
     header = StringUtils.format("Basic %s", header);
 
     HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class);
@@ -165,9 +158,7 @@ public class BasicHTTPAuthenticatorTest
   @Test
   public void testRecognizedButMalformedBasicAuthHeader() throws IOException, ServletException
   {
-    String header = Base64.encode(
+    String header = StringUtils.utf8Base64("malformed decoded header data");
-        StringUtils.toUtf8("malformed decoded header data")
-    );
     header = StringUtils.format("Basic %s", header);
 
     HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class);
@@ -214,9 +205,7 @@ public class BasicHTTPAuthenticatorTest
   @Test
   public void testUnrecognizedHeader() throws IOException, ServletException
   {
-    String header = Base64.encode(
+    String header = StringUtils.utf8Base64("userA:helloworld");
-        StringUtils.toUtf8("userA:helloworld")
-    );
     header = StringUtils.format("NotBasic %s", header);
 
     HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class);

java-util/src/main/java/org/apache/druid/java/util/common/StringUtils.java

@@ -28,6 +28,7 @@ import java.net.URLEncoder;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.IllegalFormatException;
 import java.util.Locale;
 
@@ -218,4 +219,16 @@ public class StringUtils
     return Strings.emptyToNull(string);
     //CHECKSTYLE.ON: Regexp
   }
+
+  /**
+   * Convert an input to base 64 and return the utf8 string of that byte array
+   *
+   * @param input The string to convert to base64
+   *
+   * @return the base64 of the input in string form
+   */
+  public static String utf8Base64(String input)
+  {
+    return fromUtf8(Base64.getEncoder().encode(toUtf8(input)));
+  }
 }

java-util/src/test/java/org/apache/druid/java/util/emitter/core/EmitterTest.java

@@ -31,7 +31,6 @@ import org.apache.druid.java.util.common.CompressionUtils;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.lifecycle.Lifecycle;
 import org.apache.druid.java.util.emitter.service.UnitEvent;
-import org.asynchttpclient.DefaultAsyncHttpClientConfig;
 import org.asynchttpclient.ListenableFuture;
 import org.asynchttpclient.Request;
 import org.asynchttpclient.Response;
@@ -53,6 +52,7 @@ import java.util.List;
 import java.util.Properties;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
 
 /**
  */
@@ -60,25 +60,33 @@ public class EmitterTest
 {
   private static final ObjectMapper jsonMapper = new ObjectMapper();
   public static String TARGET_URL = "http://metrics.foo.bar/";
-  public static final Response OK_RESPONSE = responseBuilder(HttpVersion.HTTP_1_1, HttpResponseStatus.CREATED)
+  public static final Response OK_RESPONSE = Stream
-      .accumulate(new EagerResponseBodyPart(Unpooled.wrappedBuffer("Yay".getBytes(StandardCharsets.UTF_8)), true))
+      .of(responseBuilder(HttpVersion.HTTP_1_1, HttpResponseStatus.CREATED))
-      .build();
+      .map(b -> {
+        b.accumulate(new EagerResponseBodyPart(Unpooled.wrappedBuffer("Yay".getBytes(StandardCharsets.UTF_8)), true));
+        return b.build();
+      }).findFirst().get();
 
-  public static final Response BAD_RESPONSE = responseBuilder(HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN)
+  public static final Response BAD_RESPONSE = Stream
-      .accumulate(new EagerResponseBodyPart(Unpooled.wrappedBuffer("Not yay".getBytes(StandardCharsets.UTF_8)), true))
+      .of(responseBuilder(HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN))
-      .build();
+      .map(b -> {
+        b.accumulate(new EagerResponseBodyPart(
+            Unpooled.wrappedBuffer("Not yay".getBytes(StandardCharsets.UTF_8)),
+            true
+        ));
+        return b.build();
+      }).findFirst().get();
 
   private static Response.ResponseBuilder responseBuilder(HttpVersion version, HttpResponseStatus status)
   {
-    return new Response.ResponseBuilder()
+    final Response.ResponseBuilder builder = new Response.ResponseBuilder();
-        .accumulate(
+    builder.accumulate(
         new NettyResponseStatus(
             Uri.create(TARGET_URL),
-                new DefaultAsyncHttpClientConfig.Builder().build(),
             new DefaultHttpResponse(version, status),
             null
-            )
+        ));
-        );
+    return builder;
   }
 
 

pom.xml

@@ -77,9 +77,8 @@
         <log4j.version>2.5</log4j.version>
         <!-- HttpClient has not yet been ported to Netty 4.x -->
         <netty3.version>3.10.6.Final</netty3.version>
-        <!-- Update to Netty 4.1 is not possible yet, see https://github.com/apache/incubator-druid/issues/4390 and comments
+        <!-- Spark updated in https://github.com/apache/spark/pull/19884 -->
-             in https://github.com/apache/incubator-druid/pull/4973 -->
+        <netty4.version>4.1.30.Final</netty4.version>
-        <netty4.version>4.0.52.Final</netty4.version>
         <slf4j.version>1.7.12</slf4j.version>
         <!-- If compiling with different hadoop version also modify default hadoop coordinates in TaskConfig.java -->
         <hadoop.compile.version>2.8.3</hadoop.compile.version>
@@ -680,7 +679,8 @@
             <dependency>
                 <groupId>org.asynchttpclient</groupId>
                 <artifactId>async-http-client</artifactId>
-                <version>2.0.37</version>
+                <!-- Uses Netty 4.1.x -->
+                <version>2.5.3</version>
             </dependency>
             <dependency>
                 <groupId>org.gridkit.lab</groupId>