BWC changes for x-pack (elastic/x-pack-elasticsearch#3841)

relates elastic/elasticsearch#28505

Original commit: elastic/x-pack-elasticsearch@eda97ded76

build.gradle

@@ -75,11 +75,12 @@ subprojects {
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-upgrade:${version}": xpackModule('upgrade')]
   ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-watcher:${version}": xpackModule('watcher')]
 
-  for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
+  bwcVersions.snapshotProjectNames.each { snapshotName ->
-    if (version.branch != null) {
+    Version snapshot = bwcVersions.getSnapshotForProject(snapshotName)
-      final String snapshotProject = ":x-pack-elasticsearch:plugin:bwc-snapshot-dummy-projects:bwc-snapshot-${version.branch}"
+    if (snapshot != null ) {
-      project(snapshotProject).ext.bwcVersion = version
+      String snapshotProject = ":x-pack-elasticsearch:plugin:bwc:${snapshotName}"
-      ext.projectSubstitutions["org.elasticsearch.plugin:x-pack:${version}"] = snapshotProject
+      project(snapshotProject).ext.bwcVersion = snapshot
+        ext.projectSubstitutions["org.elasticsearch.plugin:x-pack:${snapshot}"] = snapshotProject
     }
   }
 }

plugin/bwc-snapshot-dummy-projects/build.gradle

@@ -1,220 +0,0 @@
-import org.apache.tools.ant.taskdefs.condition.Os
-import org.elasticsearch.gradle.Version
-
-import java.util.regex.Matcher
-import org.elasticsearch.gradle.LoggedExec
-import org.elasticsearch.gradle.test.NodeInfo
-
-/**
- * Subdirectories of this project are dummy projects which does a local
- * checkout of the appropriate version's branch, and builds a snapshot. This
- * allows backcompat tests to test against the next unreleased versions
- * without relying on snapshots.
- */
-
-subprojects {
-  final Matcher match = project.name =~ /bwc-snapshot-(\d+\.(\d+|x))/
-  if (!match.matches()) {
-    throw new InvalidUserDataException("Unsupport project name ${project.name}")
-  }
-  String bwcBranch = match.group(1)
-
-  if (project.hasProperty('bwcVersion')) {
-    Version bwcVersion = project.ext.bwcVersion
-
-    apply plugin: 'distribution'
-    // Not published so no need to assemble
-    tasks.remove(assemble)
-    build.dependsOn.remove('assemble')
-
-    File esCheckoutDir = file("${buildDir}/bwc/checkout-es-${bwcBranch}")
-    /* Delay building the path as the path will not exist during configuration which will
-     * fail on Windows due to getting the short name requiring the path to already exist.
-     */
-    Object esCheckoutPath = """${->
-      if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-        esCheckoutDir.mkdirs()
-        NodeInfo.getShortPathName(esCheckoutDir.toString())
-      } else {
-        esCheckoutDir.toString()
-      }
-    }"""
-    File xpackCheckoutDir = file("${esCheckoutDir}-extra/x-pack-elasticsearch")
-    Object xpackCheckoutPath = """${->
-      if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-        xpackCheckoutDir.mkdirs()
-        NodeInfo.getShortPathName(xpackCheckoutDir.toString())
-      } else {
-        xpackCheckoutDir.toString()
-      }
-    }"""
-
-    final String remote = System.getProperty("tests.bwc.remote", "elastic")
-
-    task createElasticsearchClone(type: LoggedExec) {
-      onlyIf { esCheckoutDir.exists() == false }
-      commandLine = ['git', 'clone', rootDir, esCheckoutPath]
-    }
-
-    task createXPackClone(type: LoggedExec) {
-      onlyIf { xpackCheckoutDir.exists() == false }
-      commandLine = ['git', 'clone', xpackRootProject.projectDir, xpackCheckoutPath]
-    }
-
-    // we use regular Exec here to ensure we always get output, regardless of logging level
-    task findElasticsearchRemote(type: Exec) {
-      dependsOn createElasticsearchClone
-      workingDir = esCheckoutDir
-      commandLine = ['git', 'remote', '-v']
-      ignoreExitValue = true
-      ByteArrayOutputStream output = new ByteArrayOutputStream()
-      standardOutput = output
-      doLast {
-        if (execResult.exitValue != 0) {
-          output.toString('UTF-8').eachLine { line -> logger.error(line) }
-          execResult.assertNormalExitValue()
-        }
-        project.ext.esRemoteExists = false
-        output.toString('UTF-8').eachLine {
-          if (it.contains("${remote}\tgit@github.com:${remote}/elasticsearch.git")) {
-            project.ext.esRemoteExists = true
-          }
-        }
-      }
-    }
-
-    task findXPackRemote(type: Exec) {
-      dependsOn createXPackClone
-      workingDir = xpackCheckoutDir
-      commandLine = ['git', 'remote', '-v']
-      ignoreExitValue = true
-      ByteArrayOutputStream output = new ByteArrayOutputStream()
-      standardOutput = output
-      doLast {
-        if (execResult.exitValue != 0) {
-          output.toString('UTF-8').eachLine { line -> logger.error(line) }
-          execResult.assertNormalExitValue()
-        }
-        project.ext.xpackRemoteExists = false
-        output.toString('UTF-8').eachLine {
-          if (it.contains("${remote}\tgit@github.com:${remote}/x-pack-elasticsearch.git")) {
-            project.ext.xpackRemoteExists = true
-          }
-        }
-      }
-    }
-
-    task addElasticsearchRemote(type: LoggedExec) {
-      dependsOn findElasticsearchRemote
-      onlyIf { project.ext.esRemoteExists == false }
-      workingDir = esCheckoutDir
-      commandLine = ['git', 'remote', 'add', "${remote}", "git@github.com:${remote}/elasticsearch.git"]
-    }
-
-    task addXPackRemote(type: LoggedExec) {
-      dependsOn findXPackRemote
-      onlyIf { project.ext.xpackRemoteExists == false }
-      workingDir = xpackCheckoutDir
-      commandLine = ['git', 'remote', 'add', "${remote}", "git@github.com:${remote}/x-pack-elasticsearch.git"]
-    }
-
-    task fetchElasticsearchLatest(type: LoggedExec) {
-      dependsOn addElasticsearchRemote
-      workingDir = esCheckoutDir
-      commandLine = ['git', 'fetch', '--all']
-    }
-
-    task fetchXPackLatest(type: LoggedExec) {
-      dependsOn addXPackRemote
-      workingDir = xpackCheckoutDir
-      commandLine = ['git', 'fetch', '--all']
-    }
-
-    String esBuildMetadataKey = "bwc_refspec_${project.path.substring(1)}_elasticsearch"
-    task checkoutElasticsearchBwcBranch(type: LoggedExec) {
-      dependsOn fetchElasticsearchLatest
-      def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(esBuildMetadataKey, "${remote}/${bwcBranch}"))
-      workingDir = esCheckoutDir
-      commandLine = ['git', 'checkout', refspec]
-    }
-
-    String xpackBuildMetadataKey = "bwc_refspec_${project.path.substring(1)}_xpack"
-    task checkoutXPackBwcBranch(type: LoggedExec) {
-      dependsOn fetchXPackLatest
-      def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(xpackBuildMetadataKey, "${remote}/${bwcBranch}"))
-      workingDir = xpackCheckoutDir
-      commandLine = ['git', 'checkout', refspec]
-    }
-
-    File esBuildMetadataFile = project.file("build/${project.name}_elasticsearch/build_metadata")
-    task writeElasticsearchBuildMetadata(type: LoggedExec) {
-      dependsOn checkoutElasticsearchBwcBranch
-      workingDir = esCheckoutDir
-      commandLine = ['git', 'rev-parse', 'HEAD']
-      ignoreExitValue = true
-      ByteArrayOutputStream output = new ByteArrayOutputStream()
-      standardOutput = output
-      doLast {
-        if (execResult.exitValue != 0) {
-          output.toString('UTF-8').eachLine { line -> logger.error(line) }
-          execResult.assertNormalExitValue()
-        }
-        project.mkdir(esBuildMetadataFile.parent)
-        esBuildMetadataFile.setText("${esBuildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8')
-      }
-    }
-
-    File xpackBuildMetadataFile = project.file("build/${project.name}_xpack/build_metadata")
-    task writeXPackBuildMetadata(type: LoggedExec) {
-      dependsOn checkoutXPackBwcBranch
-      workingDir = xpackCheckoutDir
-      commandLine = ['git', 'rev-parse', 'HEAD']
-      ignoreExitValue = true
-      ByteArrayOutputStream output = new ByteArrayOutputStream()
-      standardOutput = output
-      doLast {
-        if (execResult.exitValue != 0) {
-          output.toString('UTF-8').eachLine { line -> logger.error(line) }
-          execResult.assertNormalExitValue()
-        }
-        project.mkdir(xpackBuildMetadataFile.parent)
-        xpackBuildMetadataFile.setText("${xpackBuildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8')
-      }
-    }
-
-    File bwcZip = file("${xpackCheckoutDir}/plugin/build/distributions/x-pack-${bwcVersion}.zip")
-    task buildBwcVersion(type: Exec) {
-      dependsOn checkoutXPackBwcBranch, checkoutElasticsearchBwcBranch, writeElasticsearchBuildMetadata, writeXPackBuildMetadata
-      workingDir = xpackCheckoutDir
-      if (project.rootProject.ext.runtimeJavaVersion == JavaVersion.VERSION_1_8 && ["5.6", "6.0", "6.1"].contains(bwcBranch)) {
-        /*
-         * If runtime Java home is set to JDK 8 and we are building branches that are officially built with JDK 8, push this to JAVA_HOME for
-         * these builds.
-         */
-        environment('JAVA_HOME', System.getenv('RUNTIME_JAVA_HOME'))
-      }
-      if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-        executable 'cmd'
-        args '/C', 'call', new File(xpackCheckoutDir, 'gradlew').toString()
-      } else {
-        executable new File(xpackCheckoutDir, 'gradlew').toString()
-      }
-      args ":x-pack-elasticsearch:plugin:assemble", "-Dbuild.snapshot=${System.getProperty('build.snapshot') ?: 'true'}"
-      final LogLevel logLevel = gradle.startParameter.logLevel
-      if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
-        args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
-      }
-      final String showStacktraceName = gradle.startParameter.showStacktrace.name()
-      assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
-      if (showStacktraceName.equals("ALWAYS")) {
-        args "--stacktrace"
-      } else if (showStacktraceName.equals("ALWAYS_FULL")) {
-        args "--full-stacktrace"
-      }
-    }
-
-    artifacts {
-      'default' file: bwcZip, name: 'x-pack', type: 'zip', builtBy: buildBwcVersion
-    }
-  }
-}

plugin/bwc/build.gradle

@@ -0,0 +1,225 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.Version
+
+import java.util.regex.Matcher
+import org.elasticsearch.gradle.LoggedExec
+import org.elasticsearch.gradle.test.NodeInfo
+
+/**
+ * Subdirectories of this project are dummy projects which does a local
+ * checkout of the appropriate version's branch, and builds a snapshot. This
+ * allows backcompat tests to test against the next unreleased versions
+ * without relying on snapshots.
+ */
+
+subprojects {
+
+  Version bwcVersion = bwcVersions.getSnapshotForProject(project.name)
+  if (bwcVersion == null) {
+    // this project wont do anything
+    return
+  }
+
+  String bwcBranch
+  if (project.name == 'next-minor-snapshot') {
+    // this is always a .x series
+    bwcBranch = "${bwcVersion.major}.x"
+  } else {
+    bwcBranch = "${bwcVersion.major}.${bwcVersion.minor}"
+  }
+
+  apply plugin: 'distribution'
+  // Not published so no need to assemble
+  tasks.remove(assemble)
+  build.dependsOn.remove('assemble')
+
+  File esCheckoutDir = file("${buildDir}/bwc/checkout-es-${bwcBranch}")
+  /* Delay building the path as the path will not exist during configuration which will
+   * fail on Windows due to getting the short name requiring the path to already exist.
+   */
+  Object esCheckoutPath = """${->
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+      esCheckoutDir.mkdirs()
+      NodeInfo.getShortPathName(esCheckoutDir.toString())
+    } else {
+      esCheckoutDir.toString()
+    }
+  }"""
+  File xpackCheckoutDir = file("${esCheckoutDir}-extra/x-pack-elasticsearch")
+  Object xpackCheckoutPath = """${->
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+      xpackCheckoutDir.mkdirs()
+      NodeInfo.getShortPathName(xpackCheckoutDir.toString())
+    } else {
+      xpackCheckoutDir.toString()
+    }
+  }"""
+
+  final String remote = System.getProperty("tests.bwc.remote", "elastic")
+
+  task createElasticsearchClone(type: LoggedExec) {
+    onlyIf { esCheckoutDir.exists() == false }
+    commandLine = ['git', 'clone', rootDir, esCheckoutPath]
+  }
+
+  task createXPackClone(type: LoggedExec) {
+    onlyIf { xpackCheckoutDir.exists() == false }
+    commandLine = ['git', 'clone', xpackRootProject.projectDir, xpackCheckoutPath]
+  }
+
+  // we use regular Exec here to ensure we always get output, regardless of logging level
+  task findElasticsearchRemote(type: Exec) {
+    dependsOn createElasticsearchClone
+    workingDir = esCheckoutDir
+    commandLine = ['git', 'remote', '-v']
+    ignoreExitValue = true
+    ByteArrayOutputStream output = new ByteArrayOutputStream()
+    standardOutput = output
+    doLast {
+      if (execResult.exitValue != 0) {
+        output.toString('UTF-8').eachLine { line -> logger.error(line) }
+        execResult.assertNormalExitValue()
+      }
+      project.ext.esRemoteExists = false
+      output.toString('UTF-8').eachLine {
+        if (it.contains("${remote}\tgit@github.com:${remote}/elasticsearch.git")) {
+          project.ext.esRemoteExists = true
+        }
+      }
+    }
+  }
+
+  task findXPackRemote(type: Exec) {
+    dependsOn createXPackClone
+    workingDir = xpackCheckoutDir
+    commandLine = ['git', 'remote', '-v']
+    ignoreExitValue = true
+    ByteArrayOutputStream output = new ByteArrayOutputStream()
+    standardOutput = output
+    doLast {
+      if (execResult.exitValue != 0) {
+        output.toString('UTF-8').eachLine { line -> logger.error(line) }
+        execResult.assertNormalExitValue()
+      }
+      project.ext.xpackRemoteExists = false
+      output.toString('UTF-8').eachLine {
+        if (it.contains("${remote}\tgit@github.com:${remote}/x-pack-elasticsearch.git")) {
+          project.ext.xpackRemoteExists = true
+        }
+      }
+    }
+  }
+
+  task addElasticsearchRemote(type: LoggedExec) {
+    dependsOn findElasticsearchRemote
+    onlyIf { project.ext.esRemoteExists == false }
+    workingDir = esCheckoutDir
+    commandLine = ['git', 'remote', 'add', "${remote}", "git@github.com:${remote}/elasticsearch.git"]
+  }
+
+  task addXPackRemote(type: LoggedExec) {
+    dependsOn findXPackRemote
+    onlyIf { project.ext.xpackRemoteExists == false }
+    workingDir = xpackCheckoutDir
+    commandLine = ['git', 'remote', 'add', "${remote}", "git@github.com:${remote}/x-pack-elasticsearch.git"]
+  }
+
+  task fetchElasticsearchLatest(type: LoggedExec) {
+    dependsOn addElasticsearchRemote
+    workingDir = esCheckoutDir
+    commandLine = ['git', 'fetch', '--all']
+  }
+
+  task fetchXPackLatest(type: LoggedExec) {
+    dependsOn addXPackRemote
+    workingDir = xpackCheckoutDir
+    commandLine = ['git', 'fetch', '--all']
+  }
+
+  String esBuildMetadataKey = "bwc_refspec_${project.path.substring(1)}_elasticsearch"
+  task checkoutElasticsearchBwcBranch(type: LoggedExec) {
+    dependsOn fetchElasticsearchLatest
+    def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(esBuildMetadataKey, "${remote}/${bwcBranch}"))
+    workingDir = esCheckoutDir
+    commandLine = ['git', 'checkout', refspec]
+  }
+
+  String xpackBuildMetadataKey = "bwc_refspec_${project.path.substring(1)}_xpack"
+  task checkoutXPackBwcBranch(type: LoggedExec) {
+    dependsOn fetchXPackLatest
+    def String refspec = System.getProperty("tests.bwc.refspec", buildMetadata.get(xpackBuildMetadataKey, "${remote}/${bwcBranch}"))
+    workingDir = xpackCheckoutDir
+    commandLine = ['git', 'checkout', refspec]
+  }
+
+  File esBuildMetadataFile = project.file("build/${project.name}_elasticsearch/build_metadata")
+  task writeElasticsearchBuildMetadata(type: LoggedExec) {
+    dependsOn checkoutElasticsearchBwcBranch
+    workingDir = esCheckoutDir
+    commandLine = ['git', 'rev-parse', 'HEAD']
+    ignoreExitValue = true
+    ByteArrayOutputStream output = new ByteArrayOutputStream()
+    standardOutput = output
+    doLast {
+      if (execResult.exitValue != 0) {
+        output.toString('UTF-8').eachLine { line -> logger.error(line) }
+        execResult.assertNormalExitValue()
+      }
+      project.mkdir(esBuildMetadataFile.parent)
+      esBuildMetadataFile.setText("${esBuildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8')
+    }
+  }
+
+  File xpackBuildMetadataFile = project.file("build/${project.name}_xpack/build_metadata")
+  task writeXPackBuildMetadata(type: LoggedExec) {
+    dependsOn checkoutXPackBwcBranch
+    workingDir = xpackCheckoutDir
+    commandLine = ['git', 'rev-parse', 'HEAD']
+    ignoreExitValue = true
+    ByteArrayOutputStream output = new ByteArrayOutputStream()
+    standardOutput = output
+    doLast {
+      if (execResult.exitValue != 0) {
+        output.toString('UTF-8').eachLine { line -> logger.error(line) }
+        execResult.assertNormalExitValue()
+      }
+      project.mkdir(xpackBuildMetadataFile.parent)
+      xpackBuildMetadataFile.setText("${xpackBuildMetadataKey}=${output.toString('UTF-8')}", 'UTF-8')
+    }
+  }
+
+  File bwcZip = file("${xpackCheckoutDir}/plugin/build/distributions/x-pack-${bwcVersion}.zip")
+  task buildBwcVersion(type: Exec) {
+    dependsOn checkoutXPackBwcBranch, checkoutElasticsearchBwcBranch, writeElasticsearchBuildMetadata, writeXPackBuildMetadata
+    workingDir = xpackCheckoutDir
+    if (project.rootProject.ext.runtimeJavaVersion == JavaVersion.VERSION_1_8 && ["5.6", "6.0", "6.1"].contains(bwcBranch)) {
+      /*
+       * If runtime Java home is set to JDK 8 and we are building branches that are officially built with JDK 8, push this to JAVA_HOME for
+       * these builds.
+       */
+      environment('JAVA_HOME', System.getenv('RUNTIME_JAVA_HOME'))
+    }
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+      executable 'cmd'
+      args '/C', 'call', new File(xpackCheckoutDir, 'gradlew').toString()
+    } else {
+      executable new File(xpackCheckoutDir, 'gradlew').toString()
+    }
+    args ":x-pack-elasticsearch:plugin:assemble", "-Dbuild.snapshot=${System.getProperty('build.snapshot') ?: 'true'}"
+    final LogLevel logLevel = gradle.startParameter.logLevel
+    if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
+      args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
+    }
+    final String showStacktraceName = gradle.startParameter.showStacktrace.name()
+    assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
+    if (showStacktraceName.equals("ALWAYS")) {
+      args "--stacktrace"
+    } else if (showStacktraceName.equals("ALWAYS_FULL")) {
+      args "--full-stacktrace"
+    }
+  }
+
+  artifacts {
+    'default' file: bwcZip, name: 'x-pack', type: 'zip', builtBy: buildBwcVersion
+  }
+}

qa/full-cluster-restart/build.gradle

@@ -130,7 +130,7 @@ subprojects {
     into outputDir
   }
 
-  for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
+  for (Version version : bwcVersions.indexCompatible) {
     String baseName = "v${version}"
 
     Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
@@ -239,7 +239,7 @@ subprojects {
   // basic integ tests includes testing bwc against the most recent version
   task integTest {
     if (project.bwc_tests_enabled) {
-      for (final def version : versionCollection.basicIntegrationTestVersions) {
+      for (final def version : bwcVersions.snapshotsIndexCompatible) {
         dependsOn "v${version}#bwcTest"
       }
     }

qa/rolling-upgrade/build.gradle

@@ -105,7 +105,7 @@ subprojects {
     into outputDir
   }
 
-  for (Version version : versionCollection.versionsWireCompatibleWithCurrent) {
+  for (Version version : bwcVersions.wireCompatible) {
     String baseName = "v${version}"
 
     Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
@@ -258,7 +258,7 @@ subprojects {
   // basic integ tests includes testing bwc against the most recent version
   task integTest {
     if (project.bwc_tests_enabled) {
-      for (final def version : versionCollection.basicIntegrationTestVersions) {
+      for (final def version : bwcVersions.snapshotsWireCompatible) {
         dependsOn "v${version}#bwcTest"
       }
     }