From 4605c323ccd97de14b5718c980011a1200398c24 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Mon, 7 Nov 2022 23:14:26 +0800 Subject: [PATCH] HBASE-27443 Use java11 in the general check of our jenkins job (#4845) Signed-off-by: Guanghao Zhang (cherry picked from commit 5c6ff7dc47b5340ff02e0dec144424ab91407e07) --- dev-support/Jenkinsfile | 4 +- dev-support/Jenkinsfile_GitHub | 2 +- hbase-build-configuration/pom.xml | 53 ++++++++----------- .../hadoop/hbase/util/BloomFilterFactory.java | 2 +- .../hadoop/hbase/util/ServerCommandLine.java | 14 ++--- .../hadoop/hbase/client/TestMetaCache.java | 16 ++---- .../TestFavoredNodeAssignmentHelper.java | 13 +++-- .../hbase/regionserver/TestScanner.java | 15 +++--- .../TestNonHBaseReplicationEndpoint.java | 11 ++-- .../regionserver/TestBasicWALEntryStream.java | 6 ++- .../TestReplicationSourceManager.java | 6 ++- pom.xml | 2 +- 12 files changed, 63 insertions(+), 81 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 2098fff8008..8bbb00ebbb9 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -202,7 +202,9 @@ pipeline { environment { BASEDIR = "${env.WORKSPACE}/component" TESTS = "${env.SHALLOW_CHECKS}" - SET_JAVA_HOME = '/usr/lib/jvm/java-8' + SET_JAVA_HOME = "/usr/lib/jvm/java-11" + // Activates hadoop 3.0 profile in maven runs. + HADOOP_PROFILE = '3.0' OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}" OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}" ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub index 351f1f30b54..ac548757df2 100644 --- a/dev-support/Jenkinsfile_GitHub +++ b/dev-support/Jenkinsfile_GitHub @@ -78,7 +78,7 @@ pipeline { environment { // customized per parallel stage PLUGINS = "${GENERAL_CHECK_PLUGINS}" - SET_JAVA_HOME = '/usr/lib/jvm/java-8' + SET_JAVA_HOME = "/usr/lib/jvm/java-11" WORKDIR_REL = "${WORKDIR_REL_GENERAL_CHECK}" // identical for all parallel stages WORKDIR = "${WORKSPACE}/${WORKDIR_REL}" diff --git a/hbase-build-configuration/pom.xml b/hbase-build-configuration/pom.xml index e9bb67205fc..66dc459ced8 100644 --- a/hbase-build-configuration/pom.xml +++ b/hbase-build-configuration/pom.xml @@ -69,31 +69,6 @@ false - - - - 9+181-r4173-1 - - - - com.google.errorprone - error_prone_core - ${error-prone.version} - provided - - - com.google.code.findbugs - jsr305 - - - - - com.google.errorprone - javac - ${javac.version} - provided - - @@ -101,17 +76,12 @@ org.apache.maven.plugins maven-compiler-plugin - ${compileSource} - ${compileSource} - - true + ${releaseTarget} true -XDcompilePolicy=simple - -Xplugin:ErrorProne -XepDisableWarningsInGeneratedCode -Xep:FallThrough:OFF -Xep:MutablePublicArray:OFF -Xep:ClassNewInstance:ERROR -Xep:MissingDefault:ERROR - - -J-Xbootclasspath/p:${settings.localRepository}/com/google/errorprone/javac/${javac.version}/javac-${javac.version}.jar + -Xplugin:ErrorProne -XepDisableWarningsInGeneratedCode -XepExcludedPaths:.*/target/.* -Xep:FallThrough:OFF -Xep:MutablePublicArray:OFF -Xep:ClassNewInstance:ERROR -Xep:MissingDefault:ERROR -Xep:BanJNDI:WARN @@ -122,6 +92,25 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + jdk11-required + + enforce + + + + + [11,) + + + + + + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java index 806c615ea57..08f5cb5df33 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java @@ -195,4 +195,4 @@ public final class BloomFilterFactory { writer.addInlineBlockWriter(bloomWriter); return bloomWriter; } -}; +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java index b6badc79097..b257a938337 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerCommandLine.java @@ -36,21 +36,17 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + /** * Base class for command lines that start up various HBase daemons. */ @InterfaceAudience.Private public abstract class ServerCommandLine extends Configured implements Tool { private static final Logger LOG = LoggerFactory.getLogger(ServerCommandLine.class); - @SuppressWarnings("serial") - private static final Set DEFAULT_SKIP_WORDS = new HashSet() { - { - add("secret"); - add("passwd"); - add("password"); - add("credential"); - } - }; + + private static final Set DEFAULT_SKIP_WORDS = + ImmutableSet.of("secret", "passwd", "password", "credential"); /** * Implementing subclasses should return a usage string to print out. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java index a7897ae1c61..777bb24d92b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; -import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CallQueueTooBigException; @@ -266,16 +266,10 @@ public class TestMetaCache { } public static List metaCachePreservingExceptions() { - return new ArrayList() { - { - add(new RegionOpeningException(" ")); - add(new RegionTooBusyException("Some old message")); - add(new RpcThrottlingException(" ")); - add(new MultiActionResultTooLarge(" ")); - add(new RetryImmediatelyException(" ")); - add(new CallQueueTooBigException()); - } - }; + return Arrays.asList(new RegionOpeningException(" "), + new RegionTooBusyException("Some old message"), new RpcThrottlingException(" "), + new MultiActionResultTooLarge(" "), new RetryImmediatelyException(" "), + new CallQueueTooBigException()); } public static class RegionServerWithFakeRpcServices extends HRegionServer { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/favored/TestFavoredNodeAssignmentHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/favored/TestFavoredNodeAssignmentHelper.java index d594cb6b9c4..f3e437ec02f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/favored/TestFavoredNodeAssignmentHelper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/favored/TestFavoredNodeAssignmentHelper.java @@ -364,13 +364,12 @@ public class TestFavoredNodeAssignmentHelper { regionMap.put(regionsOnRack1, 1); regionMap.put(regionsOnRack2, 2); regionMap.put(regionsOnRack3, 3); - assertTrue(printProportions(firstRackSize, secondRackSize, thirdRackSize, regionsOnRack1, - regionsOnRack2, regionsOnRack3), rackMap.get(firstRackSize) == regionMap.get(regionsOnRack1)); - assertTrue(printProportions(firstRackSize, secondRackSize, thirdRackSize, regionsOnRack1, - regionsOnRack2, regionsOnRack3), - rackMap.get(secondRackSize) == regionMap.get(regionsOnRack2)); - assertTrue(printProportions(firstRackSize, secondRackSize, thirdRackSize, regionsOnRack1, - regionsOnRack2, regionsOnRack3), rackMap.get(thirdRackSize) == regionMap.get(regionsOnRack3)); + assertEquals(printProportions(firstRackSize, secondRackSize, thirdRackSize, regionsOnRack1, + regionsOnRack2, regionsOnRack3), rackMap.get(firstRackSize), regionMap.get(regionsOnRack1)); + assertEquals(printProportions(firstRackSize, secondRackSize, thirdRackSize, regionsOnRack1, + regionsOnRack2, regionsOnRack3), rackMap.get(secondRackSize), regionMap.get(regionsOnRack2)); + assertEquals(printProportions(firstRackSize, secondRackSize, thirdRackSize, regionsOnRack1, + regionsOnRack2, regionsOnRack3), rackMap.get(thirdRackSize), regionMap.get(regionsOnRack3)); } private String printProportions(int firstRackSize, int secondRackSize, int thirdRackSize, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java index d6edb1cf7c4..941d1662804 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java @@ -538,14 +538,15 @@ public class TestScanner { } } - /* - * @param hri Region + /** + * Count table. + * @param hri Region * @param flushIndex At what row we start the flush. * @param concurrent if the flush should be concurrent or sync. * @return Count of rows found. */ private int count(final Table countTable, final int flushIndex, boolean concurrent) - throws IOException { + throws Exception { LOG.info("Taking out counting scan"); Scan scan = new Scan(); for (byte[] qualifier : EXPLICIT_COLS) { @@ -573,10 +574,10 @@ public class TestScanner { } } }; - if (concurrent) { - t.start(); // concurrently flush. - } else { - t.run(); // sync flush + t.start(); + if (!concurrent) { + // sync flush + t.join(); } LOG.info("Continuing on after kicking off background flush"); justFlushed = true; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNonHBaseReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNonHBaseReplicationEndpoint.java index ac2bde4d132..080489f9702 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNonHBaseReplicationEndpoint.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNonHBaseReplicationEndpoint.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.replication; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -45,6 +43,8 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; + @Category({ MediumTests.class, ReplicationTests.class }) public class TestNonHBaseReplicationEndpoint { @@ -86,11 +86,8 @@ public class TestNonHBaseReplicationEndpoint { ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder() .setReplicationEndpointImpl(NonHBaseReplicationEndpoint.class.getName()) - .setReplicateAllUserTables(false).setTableCFsMap(new HashMap>() { - { - put(tableName, new ArrayList<>()); - } - }).build(); + .setReplicateAllUserTables(false) + .setTableCFsMap(ImmutableMap.of(tableName, new ArrayList<>())).build(); ADMIN.addReplicationPeer("1", peerConfig); loadData(table); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java index 8601bff885c..eda89b232c3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestBasicWALEntryStream.java @@ -611,7 +611,8 @@ public abstract class TestBasicWALEntryStream extends WALEntryStreamTestBase { localLogQueue.enqueueLog(emptyLog, fakeWalGroupId); ReplicationSourceWALReader reader = new ReplicationSourceWALReader(fs, conf, localLogQueue, 0, getDummyFilter(), source, fakeWalGroupId); - reader.run(); + reader.start(); + reader.join(); // ReplicationSourceWALReaderThread#handleEofException method will // remove empty log from logQueue. assertEquals(0, localLogQueue.getQueueSize(fakeWalGroupId)); @@ -650,7 +651,8 @@ public abstract class TestBasicWALEntryStream extends WALEntryStreamTestBase { getDummyFilter(), source, fakeWalGroupId); assertEquals("Initial log queue size is not correct", 2, localLogQueue.getQueueSize(fakeWalGroupId)); - reader.run(); + reader.start(); + reader.join(); // remove empty log from logQueue. assertEquals(0, localLogQueue.getQueueSize(fakeWalGroupId)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java index 02bd3f6fb9c..21a7f27d727 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java @@ -381,7 +381,8 @@ public abstract class TestReplicationSourceManager { ReplicationFactory.getReplicationPeers(s1.getZooKeeper(), s1.getConfiguration()); rp1.init(); NodeFailoverWorker w1 = manager.new NodeFailoverWorker(server.getServerName()); - w1.run(); + w1.start(); + w1.join(); assertEquals(1, manager.getWalsByIdRecoveredQueues().size()); String id = "1-" + server.getServerName().getServerName(); assertEquals(files, manager.getWalsByIdRecoveredQueues().get(id).get(group)); @@ -402,7 +403,8 @@ public abstract class TestReplicationSourceManager { rq.addWAL(server.getServerName(), "2", group + ".log2"); NodeFailoverWorker w1 = manager.new NodeFailoverWorker(server.getServerName()); - w1.run(); + w1.start(); + w1.join(); // The log of the unknown peer should be removed from zk for (String peer : manager.getAllQueues()) { diff --git a/pom.xml b/pom.xml index 5ef25340221..e8c6d144af5 100755 --- a/pom.xml +++ b/pom.xml @@ -611,7 +611,7 @@ --> 8.29 1.6.0 - 2.10.0 + 2.16 2.4.2 1.0.0 1.8