Make unit tests pass with Java 21 (#15014)

This change updates dependencies as needed and fixes tests to remove code incompatible with Java 21
As a result all unit tests now pass with Java 21.

* update maven-shade-plugin to 3.5.0 and follow-up to #15042
  * explain why we need to override configuration when specifying outputFile
  * remove configuration from dependency management in favor of explicit overrides in each module.
* update to mockito to 5.5.0 for Java 21 support when running with Java 11+
  * continue using latest mockito 4.x (4.11.0) when running with Java 8  
  * remove need to mock private fields
* exclude incorrectly declared mockito dependency from pac4j-oidc
* remove mocking of ByteBuffer, since sealed classes can no longer be mocked in Java 21
* add JVM options workaround for system-rules junit plugin not supporting Java 18+
* exclude older versions of byte-buddy from assertj-core
* fix for Java 19 changes in floating point string representation
* fix missing InitializedNullHandlingTest
* update easymock to 5.2.0 for Java 21 compatibility
* update animal-sniffer-plugin to 1.23
* update nl.jqno.equalsverifier to 3.15.1
* update exec-maven-plugin to 3.1.0

benchmarks/src/test/java/org/apache/druid/benchmark/FlattenJSONBenchmarkUtilTest.java

@@ -20,6 +20,7 @@
 package org.apache.druid.benchmark;
 
 import org.apache.druid.java.util.common.parsers.Parser;
+import org.apache.druid.utils.JvmUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -64,15 +65,31 @@ public class FlattenJSONBenchmarkUtilTest
     Assert.assertEquals("129047958", event.get("e2.ad1[0]").toString());
     Assert.assertEquals("1658972185", event.get("e2.ad1[1]").toString());
     Assert.assertEquals("-997010830", event.get("e2.ad1[2]").toString());
+
+    // Java 19 changes some floating point string representation
+    // https://bugs.openjdk.org/browse/JDK-8291475
+    if (JvmUtils.majorVersion() < 19) {
       Assert.assertEquals("-5.8772014847368817E18", event.get("e3.m1").toString());
+    } else {
+      Assert.assertEquals("-5.877201484736882E18", event.get("e3.m1").toString());
+    }
+
     Assert.assertEquals("0.4375433369079904", event.get("e3.m2").toString());
     Assert.assertEquals("0.8510482953607659", event.get("e3.m3").toString());
     Assert.assertEquals("-2.3832626488759337E18", event.get("e3.m4").toString());
+    if (JvmUtils.majorVersion() < 19) {
       Assert.assertEquals("7.9789762132607068E18", event.get("e3.am1[0]").toString());
       Assert.assertEquals("-7.8634787235005573E18", event.get("e3.am1[1]").toString());
       Assert.assertEquals("8.7372945568982446E18", event.get("e3.am1[2]").toString());
       Assert.assertEquals("3.1928124802414899E18", event.get("e3.am1[3]").toString());
       Assert.assertEquals("-3.9806631713718011E18", event.get("e4.e4.m4").toString());
+    } else {
+      Assert.assertEquals("7.978976213260707E18", event.get("e3.am1[0]").toString());
+      Assert.assertEquals("-7.863478723500557E18", event.get("e3.am1[1]").toString());
+      Assert.assertEquals("8.737294556898245E18", event.get("e3.am1[2]").toString());
+      Assert.assertEquals("3.19281248024149E18", event.get("e3.am1[3]").toString());
+      Assert.assertEquals("-3.980663171371801E18", event.get("e4.e4.m4").toString());
+    }
     Assert.assertEquals("-1915243040", event.get("ae1[0].d1").toString());
     Assert.assertEquals("-2020543641", event.get("ae1[1].d1").toString());
     Assert.assertEquals("1414285347", event.get("ae1[2].e1.d2").toString());

extensions-contrib/kubernetes-overlord-extensions/pom.xml

@@ -214,7 +214,6 @@
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
-      <version>${mockito.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>

extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChestTest.java

@@ -45,6 +45,7 @@ import org.apache.druid.query.groupby.ResultRow;
 import org.apache.druid.query.timeseries.TimeseriesQuery;
 import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest;
 import org.apache.druid.query.timeseries.TimeseriesResultValue;
+import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -54,7 +55,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-public class MaterializedViewQueryQueryToolChestTest
+public class MaterializedViewQueryQueryToolChestTest extends InitializedNullHandlingTest
 {
   static {
     NullHandling.initializeForTests();

extensions-core/azure-extensions/pom.xml

@@ -154,13 +154,14 @@
         </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
+            <artifactId>mockito-${mockito.inline.artifact}</artifactId>
             <version>${mockito.version}</version>
             <scope>test</scope>
         </dependency>
+        <!-- explicitly declare mockito-core dependency to make anaylize-dependencies happy when running with Java 8 -->
         <dependency>
             <groupId>org.mockito</groupId>
-            <artifactId>mockito-inline</artifactId>
+            <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>

extensions-core/druid-pac4j/pom.xml

@@ -64,6 +64,13 @@
       <groupId>org.pac4j</groupId>
       <artifactId>pac4j-oidc</artifactId>
       <version>${pac4j.version}</version>
+      <exclusions>
+        <!-- pac4j-oidc erroneously declares mockito as a compile time instead of test dependency -->
+        <exclusion>
+          <groupId>org.mockito</groupId>
+          <artifactId>mockito-core</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>

extensions-core/lookups-cached-global/pom.xml

@@ -140,14 +140,16 @@
       <artifactId>easymock</artifactId>
       <scope>test</scope>
     </dependency>
+    <!-- explicitly declare mockito-core dependency to make anaylize-dependencies happy when running with Java 8 -->
     <dependency>
       <groupId>org.mockito</groupId>
-      <artifactId>mockito-inline</artifactId>
+      <artifactId>mockito-core</artifactId>
+      <version>${mockito.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
-      <artifactId>mockito-core</artifactId>
+      <artifactId>mockito-${mockito.inline.artifact}</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

extensions-core/protobuf-extensions/pom.xml

@@ -187,7 +187,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.4.1</version>
         <configuration>
           <createDependencyReducedPom>true</createDependencyReducedPom>
           <relocations>

indexing-hadoop/pom.xml

@@ -140,17 +140,20 @@
             <artifactId>hamcrest-core</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-inline</artifactId>
-            <scope>test</scope>
-        </dependency>
+        <!-- explicitly declare mockito-core dependency to make anaylize-dependencies happy when running with Java 8 -->
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
+            <version>${mockito.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-${mockito.inline.artifact}</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>
+
     <profiles>
         <profile>
             <id>hadoop3</id>
@@ -197,6 +200,12 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <!-- when specifying outputFile the shade plugin should not replace the main artifact.
+                                 Starting with maven-shade-plugin 3.3.0 shaded dependencies are removed in the main pom.
+                                 Setting this flag preserves the behavior of previous versions to list all
+                                 dependencies in the final pom to ensure our distribution step has the correct
+                                 classpath with all dependencies when running pull-deps -->
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <outputFile>
                                 ${project.build.directory}/${project.artifactId}-${project.version}-selfcontained.jar
                             </outputFile>

indexing-service/pom.xml

@@ -265,7 +265,6 @@
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
-            <version>${mockito.version}</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

licenses.yaml

@@ -857,27 +857,6 @@ libraries:
 
 ---
 
-name: net.bytebuddy byte-buddy
-license_category: binary
-module: extensions/druid-pac4j
-license_name: Apache License version 2.0
-version: 1.12.7
-libraries:
-  - net.bytebuddy: byte-buddy
-  - net.bytebuddy: byte-buddy-agent
-
----
-
-name: org.mockito mockito-core
-license_category: binary
-module: extensions/druid-pac4j
-license_name: MIT License
-version: 4.3.1
-libraries:
-  - org.mockito: mockito-core
-
----
-
 name: javax.activation activation
 license_category: binary
 module: extensions/druid-pac4j

pom.xml

@@ -111,7 +111,11 @@
         <jna.version>5.13.0</jna.version>
         <jna-platform.version>5.13.0</jna-platform.version>
         <hadoop.compile.version>3.3.6</hadoop.compile.version>
-        <mockito.version>4.3.1</mockito.version>
+        <mockito.version>5.5.0</mockito.version>
+        <!-- mockito-inline artifact was removed in mockito 5.3 (mockito 5.x is required for Java >17),
+             however it is required in some cases when running against mockito 4.x (mockito 4.x is required for Java <11.
+             We use the following property to pick the proper artifact based on Java version (see pre-java-11 profile) -->
+        <mockito.inline.artifact>core</mockito.inline.artifact>
         <aws.sdk.version>1.12.497</aws.sdk.version>
         <caffeine.version>2.8.0</caffeine.version>
         <jacoco.version>0.8.7</jacoco.version>
@@ -125,6 +129,7 @@
         <com.google.apis.compute.version>v1-rev20230606-2.0.0</com.google.apis.compute.version>
         <com.google.apis.storage.version>v1-rev20230301-2.0.0</com.google.apis.storage.version>
         <jdk.strong.encapsulation.argLine><!-- empty placeholder --></jdk.strong.encapsulation.argLine>
+        <jdk.security.manager.allow.argLine><!-- empty placeholder --></jdk.security.manager.allow.argLine>
         <repoOrgId>maven.org</repoOrgId>
         <repoOrgName>Maven Central Repository</repoOrgName>
         <repoOrgUrl>https://repo1.maven.org/maven2/</repoOrgUrl>
@@ -1039,7 +1044,7 @@
             <dependency>
                 <groupId>org.easymock</groupId>
                 <artifactId>easymock</artifactId>
-                <version>4.3</version>
+                <version>5.2.0</version>
                 <scope>test</scope>
             </dependency>
             <dependency>
@@ -1103,6 +1108,13 @@
                 <artifactId>assertj-core</artifactId>
                 <version>3.24.2</version>
                 <scope>test</scope>
+                <exclusions>
+                  <!-- exclude older byte-buddy until assertj-core depends on 1.14+ for Java 21 -->
+                  <exclusion>
+                    <groupId>net.bytebuddy</groupId>
+                    <artifactId>byte-buddy</artifactId>
+                  </exclusion>
+                </exclusions>
             </dependency>
             <dependency>
                 <groupId>com.ircclouds.irc</groupId>
@@ -1240,7 +1252,7 @@
             <dependency>
                 <groupId>nl.jqno.equalsverifier</groupId>
                 <artifactId>equalsverifier</artifactId>
-                <version>3.10.1</version>
+                <version>3.15.1</version>
                 <scope>test</scope>
             </dependency>
             <dependency>
@@ -1480,7 +1492,7 @@
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>animal-sniffer-maven-plugin</artifactId>
-                <version>1.17</version>
+                <version>1.23</version>
                 <executions>
                     <execution>
                         <id>check-java-api</id>
@@ -1663,6 +1675,7 @@
                         <argLine>
                             @{jacocoArgLine}
                             ${jdk.strong.encapsulation.argLine}
+                            ${jdk.security.manager.allow.argLine}
                             -Xmx1500m
                             -XX:MaxDirectMemorySize=2500m
                             -XX:+ExitOnOutOfMemoryError
@@ -1730,10 +1743,7 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-shade-plugin</artifactId>
-                    <version>3.4.1</version>
-                    <configuration>
-                        <createDependencyReducedPom>false</createDependencyReducedPom>
-                    </configuration>
+                    <version>3.5.0</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
@@ -1763,7 +1773,7 @@
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>exec-maven-plugin</artifactId>
-                    <version>1.6.0</version>
+                    <version>3.1.0</version>
                 </plugin>
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
@@ -1806,6 +1816,31 @@
     </build>
 
     <profiles>
+        <!-- mockito 5.x dropped support for Java 8, but is necessary to test against Java >17 -->
+        <profile>
+          <id>pre-java-11</id>
+          <activation>
+            <jdk>(,11)</jdk>
+          </activation>
+          <properties>
+            <!-- mockito-inline was removed in mockito 5.3, but is necessary when running against mockito 4.x for Java 8 -->
+            <mockito.version>4.11.0</mockito.version>
+            <mockito.inline.artifact>inline</mockito.inline.artifact>
+          </properties>
+        </profile>
+        <profile>
+            <id>java-12+</id>
+            <activation>
+                <jdk>[12,)</jdk>
+            </activation>
+            <properties>
+                <jdk.security.manager.allow.argLine>
+                  <!-- required for system-rules with Java >= 18 https://github.com/stefanbirkner/system-rules/issues/85 -->
+                  <!-- this option is only available starting in Java 12 -->
+                    -Djava.security.manager=allow
+                </jdk.security.manager.allow.argLine>
+            </properties>
+        </profile>
         <profile>
             <id>java-9+</id>
             <activation>

processing/pom.xml

@@ -364,9 +364,16 @@
       <artifactId>caliper</artifactId>
       <scope>test</scope>
     </dependency>
+    <!-- explicitly declare mockito-core dependency to make anaylize-dependencies happy when running with Java 8 -->
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
+      <version>${mockito.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-${mockito.inline.artifact}</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -426,11 +433,6 @@
       <artifactId>jetty-util</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-inline</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>mysql</groupId>
       <artifactId>mysql-connector-java</artifactId>

processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregatorTest.java

@@ -20,7 +20,6 @@
 package org.apache.druid.query.aggregation.constant;
 
 import org.apache.commons.lang.math.RandomUtils;
-import org.easymock.EasyMock;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -38,9 +37,8 @@ public class LongConstantBufferAggregatorTest
   {
     randomVal = RandomUtils.nextLong();
     aggregator = new LongConstantBufferAggregator(randomVal);
-    byteBuffer = EasyMock.mock(ByteBuffer.class);
-    EasyMock.replay(byteBuffer);
-    EasyMock.verifyUnexpectedCalls(byteBuffer);
+    // mark byteBuffer null to verify no methods ever get called on it.
+    byteBuffer = null;
   }
 
   @Test

processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregatorTest.java

@@ -20,7 +20,6 @@
 package org.apache.druid.query.aggregation.constant;
 
 import org.apache.commons.lang.math.RandomUtils;
-import org.easymock.EasyMock;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -38,9 +37,8 @@ public class LongConstantVectorAggregatorTest
   {
     randomVal = RandomUtils.nextLong();
     aggregator = new LongConstantVectorAggregator(randomVal);
-    byteBuffer = EasyMock.mock(ByteBuffer.class);
-    EasyMock.replay(byteBuffer);
-    EasyMock.verifyUnexpectedCalls(byteBuffer);
+    // mark byteBuffer null to verify no methods ever get called on it.
+    byteBuffer = null;
   }
 
   @Test

services/pom.xml

@@ -282,6 +282,12 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <!-- when specifying outputFile the shade plugin should not replace the main artifact.
+                                 Starting with maven-shade-plugin 3.3.0 shaded dependencies are removed in the main pom.
+                                 Setting this flag preserves the behavior of previous versions to list all
+                                 dependencies in the final pom to ensure our distribution step has the correct
+                                 classpath with all dependencies when running pull-deps -->
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
                             <outputFile>
                                 ${project.build.directory}/${project.artifactId}-${project.version}-selfcontained.jar
                             </outputFile>

sql/src/test/java/org/apache/druid/sql/calcite/rule/DruidLogicalValuesRuleTest.java

@@ -20,7 +20,6 @@
 package org.apache.druid.sql.calcite.rule;
 
 import com.google.common.collect.ImmutableList;
-import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexLiteral;
@@ -33,6 +32,7 @@ import org.apache.druid.error.DruidExceptionMatcher;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.sql.calcite.planner.DruidTypeSystem;
 import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.apache.druid.testing.InitializedNullHandlingTest;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.Assert;
@@ -44,10 +44,8 @@ import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
-import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
-import java.lang.reflect.Field;
 import java.math.BigDecimal;
 
 @RunWith(Enclosed.class)
@@ -56,7 +54,7 @@ public class DruidLogicalValuesRuleTest
   private static final PlannerContext DEFAULT_CONTEXT = Mockito.mock(PlannerContext.class);
 
   @RunWith(Parameterized.class)
-  public static class GetValueFromLiteralSimpleTypesTest
+  public static class GetValueFromLiteralSimpleTypesTest extends InitializedNullHandlingTest
   {
     @Parameters(name = "{1}, {2}")
     public static Iterable<Object[]> constructorFeeder()
@@ -88,7 +86,7 @@ public class DruidLogicalValuesRuleTest
     @Test
     public void testGetValueFromLiteral()
     {
-      final RexLiteral literal = makeLiteral(val, sqlTypeName, javaType);
+      final RexLiteral literal = Mockito.spy(makeLiteral(val, sqlTypeName, javaType));
       final Object fromLiteral = DruidLogicalValuesRule.getValueFromLiteral(literal, DEFAULT_CONTEXT);
       Assert.assertSame(javaType, fromLiteral.getClass());
       Assert.assertEquals(val, fromLiteral);
@@ -97,20 +95,11 @@ public class DruidLogicalValuesRuleTest
 
     private static RexLiteral makeLiteral(Comparable<?> val, SqlTypeName typeName, Class<?> javaType)
     {
-      RelDataType dataType = Mockito.mock(RelDataType.class);
-      Mockito.when(dataType.getSqlTypeName()).thenReturn(typeName);
-      RexLiteral literal = Mockito.mock(RexLiteral.class);
-      try {
-        Field field = literal.getClass().getSuperclass().getDeclaredField("value");
-        field.setAccessible(true);
-        field.set(literal, val);
-      }
-      catch (Exception e) {
-        Assert.fail("Unable to mock the literal for test.\nException: " + e);
-      }
-      Mockito.when(literal.getType()).thenReturn(dataType);
-      Mockito.when(literal.getValueAs(ArgumentMatchers.any())).thenReturn(javaType.cast(val));
-      return literal;
+      return (RexLiteral) new RexBuilder(DruidTypeSystem.TYPE_FACTORY).makeLiteral(
+          typeName == SqlTypeName.DECIMAL && val != null ? new BigDecimal(String.valueOf(val)) : val,
+          DruidTypeSystem.TYPE_FACTORY.createSqlType(typeName),
+          false
+      );
     }
   }