HBASE-20387 turn flaky test tracking infra into per-branch pipeline.

* gather up all the flaky test stuff into a directory
* create Jenkins Pipeline DSL for the report generation and the flaky re-testing
* have the nightly per-branch job consume the results of flaky reporting

Signed-off-by: Mike Drob <mdrob@apache.org>

dev-support/Dockerfile

@@ -17,6 +17,9 @@
 #
 # This Dockerfile is to setup environment for dev-support scripts which require
 # dependencies outside of what Apache Jenkins machines may have.
+#
+# Specifically, it's used for the flaky test reporting job defined in
+# dev-support/flaky-tests/flaky-reporting.Jenkinsfile
 FROM ubuntu:14.04
 
 ADD . /hbase/dev-support

dev-support/Jenkinsfile

@@ -47,11 +47,7 @@ pipeline {
     ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
     // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
     TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
-    // Flaky urls for different branches. Replace '-' and '.' in branch name by '_' because those
+    EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/excludes"
-    // characters are not allowed in bash variable name.
-    // Not excluding flakies from the nightly build for now.
-    // EXCLUDE_TESTS_URL_master = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
-    // EXCLUDE_TESTS_URL_branch_2 = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests-branch2.0/lastSuccessfulBuild/artifact/excludes/'
   }
   parameters {
     booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.

dev-support/flaky-tests/flaky-reporting.Jenkinsfile

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+pipeline {
+  agent {
+    node {
+      label 'Hadoop'
+    }
+  }
+  triggers {
+    cron('@daily')
+  }
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '100'))
+    timeout (time: 15, unit: 'MINUTES')
+    timestamps()
+  }
+  parameters {
+    booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
+  }
+  stages {
+    stage ('build flaky report') {
+      steps {
+        sh '''#!/usr/bin/env bash
+          set -e
+          if [ "${DEBUG}" = "true" ]; then
+            set -x
+          fi
+          declare -a flaky_args
+          flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 5)
+          flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 40)
+          docker build -t hbase-dev-support dev-support
+          docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
+'''
+      }
+    }
+  }
+  post {
+    always {
+      // Has to be relative to WORKSPACE.
+      archive "includes,excludes,dashboard.html"
+      publishHTML target: [
+        allowMissing: true,
+        keepAll: true,
+        alwaysLinkToLastBuild: true,
+        // Has to be relative to WORKSPACE
+        reportDir: ".",
+        reportFiles: 'dashboard.html',
+        reportName: 'Flaky Test Report'
+      ]
+    }
+  }
+}

dev-support/flaky-tests/run-flaky-tests.Jenkinsfile

@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+pipeline {
+  agent {
+    node {
+      label 'Hadoop'
+    }
+  }
+  triggers {
+    cron('@hourly')
+  }
+  options {
+    // this should roughly match how long we tell the flaky dashboard to look at
+    buildDiscarder(logRotator(numToKeepStr: '80'))
+    timeout (time: 2, unit: 'HOURS')
+    timestamps()
+  }
+  parameters {
+    booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
+  }
+  tools {
+    // this should match what the yetus nightly job for the branch will use
+    maven 'Maven (latest)'
+    jdk "JDK 1.8 (latest)"
+  }
+  stages {
+    stage ('run flaky tests') {
+      steps {
+        sh '''#!/usr/bin/env bash
+          set -e
+          declare -a curl_args=(--fail)
+          declare -a mvn_args=(--batch-mode -fn -Dbuild.id="${BUILD_ID}" -Dmaven.repo.local="${WORKSPACE}/local-repository")
+          if [ "${DEBUG}" = "true" ]; then
+            curl_args=("${curl_args[@]}" -v)
+            mvn_args=("${mvn_args[@]}" -X)
+            set -x
+          fi
+          ulimit -a
+          rm -rf local-repository/org/apache/hbase
+          curl "${curl_args[@]}" -o includes.txt "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/includes"
+          if [ -s includes.txt ]; then
+            mvn clean package "${mvn_args[@]}" -Dtest="$(cat includes.txt)" -Dmaven.test.redirectTestOutputToFile=true -Dsurefire.firstPartForkCount=3 -Dsurefire.secondPartForkCount=3
+          else
+            echo "set of flaky tests is currently empty."
+          fi
+'''
+      }
+    }
+  }
+  post {
+    always {
+      junit testResults: "**/surefire-reports/*.xml", allowEmptyResults: true
+      // TODO compress these logs
+      archive 'includes.txt,**/surefire-reports/*,**/test-data/*'
+    }
+  }
+}

dev-support/hbase_nightly_yetus.sh

@@ -71,10 +71,6 @@ YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--proclimit=10000" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--dockermemlimit=20g" "${YETUS_ARGS[@]}")
 
-# Currently, flaky list is calculated only for master branch.
-UNDERSCORED_BRANCH_NAME=$(echo ${BRANCH_NAME} | tr '.-' '_')
-EXCLUDE_TESTS_URL=$(eval echo "\$EXCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}")
-INCLUDE_TESTS_URL=$(eval echo "\$INCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}")
 if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
   YETUS_ARGS=("--exclude-tests-url=${EXCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
 fi