HBASE-26828 Increase the concurrency when running UTs in pre commit job (#4213)

Signed-off-by: Yulin Niu <niuyulin@apache.org>
(cherry picked from commit a49d147d49)

dev-support/Jenkinsfile_GitHub

@@ -45,7 +45,10 @@ pipeline {
         // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
         TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
         EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
-
+        // set build parallel
+        BUILD_THREAD = 4
+        SUREFIRE_FIRST_PART_FORK_COUNT = '0.5C'
+        SUREFIRE_SECOND_PART_FORK_COUNT = '0.5C'
         // a global view of paths. parallel stages can land on the same host concurrently, so each
         // stage works in its own subdirectory. there is an "output" under each of these
         // directories, which we retrieve after the build is complete.

dev-support/hbase-personality.sh

@@ -119,6 +119,18 @@ function personality_parse_args
         delete_parameter "${i}"
         ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
       ;;
+      --build-thread=*
+        delete_parameter "${i}"
+        BUILD_THREAD=${i#*=}
+      ;;
+      --surefire-first-part-fork-count=*
+        delete_parameter "${i}"
+        SUREFIRE_FIRST_PART_FORK_COUNT=${i#*=}
+      ;;
+      --surefire-second-part-fork-count=*
+        delete_parameter "${i}"
+        SUREFIRE_SECOND_PART_FORK_COUNT=${i#*=}
+      ;;
     esac
   done
 }
@@ -144,7 +156,13 @@ function personality_modules
   # At a few points, hbase modules can run build, test, etc. in parallel
   # Let it happen. Means we'll use more CPU but should be for short bursts.
   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
-  extra="--threads=2 -DHBasePatchProcess"
+  if [[ -n "${BUILD_THREAD}" ]]; then
+    extra="--threads=${BUILD_THREAD}"
+  else
+    extra="--threads=2"
+  fi
+
+  extra="${extra} -DHBasePatchProcess"
   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
     extra="${extra} -Dhttps.protocols=TLSv1.2"
   fi
@@ -232,6 +250,15 @@ function personality_modules
       extra="${extra} -Dbuild.id=${BUILD_ID}"
     fi
 
+    # set forkCount
+    if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
+      extra="${extra} -Dsurefire.firstPartForkCount=${SUREFIRE_FIRST_PART_FORK_COUNT}"
+    fi
+
+    if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
+      extra="${extra} -Dsurefire.secondPartForkCount=${SUREFIRE_SECOND_PART_FORK_COUNT}"
+    fi
+
     # If the set of changed files includes CommonFSUtils then add the hbase-server
     # module to the set of modules (if not already included) to be tested
     for f in "${CHANGED_FILES[@]}"

dev-support/jenkins_precommit_github_yetus.sh

@@ -147,6 +147,16 @@ YETUS_ARGS+=("--github-use-emoji-vote")
 if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
   YETUS_ARGS+=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}")
 fi
+# pass build parallelism in
+if [[ -n "${BUILD_THREAD}" ]]; then
+  YETUS_ARGS+=("--build-thread=${BUILD_THREAD}")
+fi
+if [[ -n "${SUREFIRE_FIRST_PART_FORK_COUNT}" ]]; then
+  YETUS_ARGS+=("--surefire-first-part-fork-count=${SUREFIRE_FIRST_PART_FORK_COUNT}")
+fi
+if [[ -n "${SUREFIRE_SECOND_PART_FORK_COUNT}" ]]; then
+  YETUS_ARGS+=("--surefire-second-part-fork-count=${SUREFIRE_SECOND_PART_FORK_COUNT}")
+fi
 
 echo "Launching yetus with command line:"
 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"