GHA steps to collect and upload heap dumps to debug UT OOM errors (#17029)

* Add GHA steps to tar and upload any heap dumps on failure to debug UT OOM issues.

* Add jvm options to heap dump OnOutOfMemoryError

Co-authored-by: Elliott Freis <108356317+imply-elliott@users.noreply.github.com>

---------

Co-authored-by: Elliott Freis <108356317+imply-elliott@users.noreply.github.com>

.github/workflows/reusable-unit-tests.yml

@@ -117,6 +117,28 @@ jobs:
           MAVEN_PROJECTS: ${{ inputs.maven_projects }}
         run: ./.github/scripts/unit_tests_script.sh
 
+      - name: Check for .hprof files on failure
+        if: ${{ failure() }}
+        id: check_for_heap_dump
+        run: |
+          if ls ${GITHUB_WORKSPACE}/target/*.hprof 1> /dev/null 2>&1; then
+            echo "found_hprof=true" >> "$GITHUB_ENV"
+          else
+            echo "found_hprof=false" >> "$GITHUB_ENV"
+          fi
+
+      - name: Collect tarball hprof dumps if they exist on failure
+        if: ${{ failure() && env.found_hprof == 'true' }}
+        run: |
+          tar cvzf ${RUNNER_TEMP}/hprof-dumps.tgz ${GITHUB_WORKSPACE}/target/*.hprof
+
+      - name: Upload hprof dumps to GitHub if they exist on failure
+        if: ${{ failure() && env.found_hprof == 'true' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: Hprof-${{ inputs.group }} hprof dumps (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }})
+          path: ${{ runner.temp }}/hprof-dumps.tgz
+
       - name: set outputs on failure
         id: set_outputs
         if: ${{ failure() }}

pom.xml

@@ -1778,6 +1778,8 @@
                             -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
                             -Daws.region=us-east-1 <!-- required for s3-related unit tests -->
                             -Ddruid.test.stupidPool.poison=true
+                            -XX:OnOutOfMemoryError='chmod 644 ${project.parent.basedir}/target/*.hprof'
+                            -XX:HeapDumpPath=${project.parent.basedir}/target
                             <!--@TODO After fixing https://github.com/apache/druid/issues/4964 remove this parameter-->
                             -Ddruid.indexing.doubleStorage=double
                             ${jfrProfilerArgLine}