Build: Add build metadata generation and usage to xpack (elastic/x-pack-elasticsearch#2368)

This commit adds generation of build_metadata files from bwc checkouts
as well as the sibling ES checkout of the xpack ci script.

relates elastic/elasticsearch#26397

Original commit: elastic/x-pack-elasticsearch@e17d904beb

dev-tools/ci

@@ -116,9 +116,32 @@ if [ -z ${USE_EXISTING_ES:+x} ]; then
 
     pick_clone_target
 
+    DEPTH=1
+    if [ -n "$BUILD_METADATA" ]; then
+      IFS=';' read -ra metadata <<< "$BUILD_METADATA"
+      for kv in "${metadata[@]}"; do
+        IFS='=' read -ra key_value <<< "$kv"
+        if [ "${key_value[0]}" == "git_ref_elasticsearch" ]; then
+          # Force checked out hash if build metadata is set. We use a depth of 100, which
+          # assumes there are no more than 100 commits between head of the branch and
+          # last-good-commit. This is still quite a bit faster than pulling the entire history.
+          ES_REF="${key_value[1]}"
+          DEPTH=100
+        fi
+      done
+    fi
+
     echo " -> checking out '$BRANCH' branch from $GH_USER/elasticsearch..."
-    git clone -b $BRANCH "https://github.com/$GH_USER/elasticsearch.git" --depth=1
+    git clone -b $BRANCH "https://github.com/$GH_USER/elasticsearch.git" --depth=$DEPTH
-    echo " -> checked out elasticsearch revision: $(git -C elasticsearch rev-parse HEAD)"
+
+    if [ ! -z $ES_REF ]; then
+      echo " -> using elasticsearch ref from build metadata: $ES_REF"
+      git -C elasticsearch checkout $ES_REF
+    else
+      ES_REF="$(git -C elasticsearch rev-parse HEAD)"
+    fi
+
+    echo " -> checked out elasticsearch revision: $ES_REF"
     echo
 
 else
@@ -145,4 +168,8 @@ gradle --stacktrace clean
 # Actually run the tests
 gradle "${GRADLE_CLI_ARGS[@]}"
 
+# write the ES hash we checked out to build metadata
+mkdir build
+echo "git_ref_elasticsearch=$ES_REF" > build/build_metadata
+
 # ~*~ shell-script-mode ~*~

plugin/bwc/build.gradle

@@ -126,23 +126,61 @@ subprojects {
       commandLine = ['git', 'fetch', '--all']
     }
 
+    String esBuildMetadataKey = "bwc_refspec_${project.path.substring(1)}_elasticsearch"
     task checkoutElasticsearchBwcBranch(type: LoggedExec) {
       dependsOn fetchElasticsearchLatest
-      def String refspec = System.getProperty("tests.bwc.refspec", "upstream/${bwcBranch}")
+      def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(esBuildMetadataKey, "upstream/${bwcBranch}"))
       workingDir = esCheckoutDir
       commandLine = ['git', 'checkout', refspec]
     }
 
+    String xpackBuildMetadataKey = "bwc_refspec_${project.path.substring(1)}_xpack"
     task checkoutXPackBwcBranch(type: LoggedExec) {
       dependsOn fetchXPackLatest
-      def String refspec = System.getProperty("tests.bwc.refspec", "upstream/${bwcBranch}")
+      def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(xpackBuildMetadataKey, "upstream/${bwcBranch}"))
       workingDir = xpackCheckoutDir
       commandLine = ['git', 'checkout', refspec]
     }
 
+    File esBuildMetadataFile = project.file("build/${project.name}_elasticsearch/build_metadata")
+    task writeElasticsearchBuildMetadata(type: LoggedExec) {
+      dependsOn checkoutElasticsearchBwcBranch
+      workingDir = esCheckoutDir
+      commandLine = ['git', 'rev-parse', 'HEAD']
+      ignoreExitValue = true
+      ByteArrayOutputStream output = new ByteArrayOutputStream()
+      standardOutput = output
+      doLast {
+        if (execResult.exitValue != 0) {
+          output.toString('UTF-8').eachLine { line -> logger.error(line) }
+          execResult.assertNormalExitValue()
+        }
+        project.mkdir(esBuildMetadataFile.parent)
+        esBuildMetadataFile.setText("${esBuildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8')
+      }
+    }
+
+    File xpackBuildMetadataFile = project.file("build/${project.name}_xpack/build_metadata")
+    task writeXPackBuildMetadata(type: LoggedExec) {
+      dependsOn checkoutXPackBwcBranch
+      workingDir = xpackCheckoutDir
+      commandLine = ['git', 'rev-parse', 'HEAD']
+      ignoreExitValue = true
+      ByteArrayOutputStream output = new ByteArrayOutputStream()
+      standardOutput = output
+      doLast {
+        if (execResult.exitValue != 0) {
+          output.toString('UTF-8').eachLine { line -> logger.error(line) }
+          execResult.assertNormalExitValue()
+        }
+        project.mkdir(xpackBuildMetadataFile.parent)
+        xpackBuildMetadataFile.setText("${xpackBuildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8')
+      }
+    }
+
     File bwcZip = file("${xpackCheckoutDir}/plugin/build/distributions/x-pack-${bwcVersion}.zip")
     task buildBwcVersion(type: GradleBuild) {
-      dependsOn checkoutXPackBwcBranch, checkoutElasticsearchBwcBranch
+      dependsOn checkoutXPackBwcBranch, checkoutElasticsearchBwcBranch, writeElasticsearchBuildMetadata, writeXPackBuildMetadata
       dir = xpackCheckoutDir
       tasks = [':x-pack-elasticsearch:plugin:assemble']
     }