Build: Extract all ES versions into gradle properties (#24748)

This commit expands the logic for version extraction from Version.java
to include a list of all versions for backcompat purposes. The tests
using bwcVersion are converted to use this list, but those tests
(rolling upgrade and backwards-5.0) are still not randomized; that will
happen in another followup.

build.gradle

@@ -61,17 +61,42 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
   }
 }
 
-int prevMajor = Integer.parseInt(VersionProperties.elasticsearch.split('\\.')[0]) - 1
-String prevSnapshot = VersionProperties.elasticsearch.contains('alpha') ? '-SNAPSHOT' : ''
+// introspect all versions of ES that may be tested agains for backwards compatibility
+String currentVersion = VersionProperties.elasticsearch.minus('-SNAPSHOT')
+int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1
 File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
 List<String> versionLines = versionFile.readLines('UTF-8')
-int prevMinor = 0
+List<String> versions = []
+// keep track of the previous major version's last minor, so we know where wire compat begins
+int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
+int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
 for (String line : versionLines) {
-  Matcher match = line =~ /\W+public static final Version V_${prevMajor}_(\d+)_.*/
+  Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_UNRELEASED)? .*/
   if (match.matches()) {
-    prevMinor = Math.max(Integer.parseInt(match.group(1)), prevMinor)
+    int major = Integer.parseInt(match.group(1))
+    int minor = Integer.parseInt(match.group(2))
+    int bugfix = Integer.parseInt(match.group(3))
+    String versionStr = "${major}.${minor}.${bugfix}"
+    if (currentVersion != versionStr) {
+      versions.add(versionStr)
+    }
+    if (major == prevMajor && minor > lastPrevMinor) {
+      prevMinorIndex = versions.size() - 1
+      lastPrevMinor = minor
+    }
   }
 }
+if (versions.toSorted() != versions) {
+  throw new GradleException('Versions.java contains out of order version constants')
+}
+if (currentVersion.split('\\.')[2].split('-')[0] == '0') {
+  // If on a release branch, after the initial release of that branch, the bugfix version will
+  // be bumped, and will be != 0. On master and N.x branches, we want to test against the
+  // unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
+  // and the bwc-zip distribution will checkout and build that version. The version parsing
+  // logic above pulls the bugfix version, and then strips off any prerelease version
+  versions[-1] += '-SNAPSHOT'
+}
 
 // injecting groovy property variables into all projects
 allprojects {
@@ -80,7 +105,8 @@ allprojects {
     isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
     isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
     // for backcompat testing
-    bwcVersion = "${prevMajor}.${prevMinor}.0${prevSnapshot}"
+    indexCompatVersions = versions
+    wireCompatVersions = versions.subList(prevMinorIndex, versions.size())
   }
 }
 
@@ -128,7 +154,7 @@ subprojects {
     "org.elasticsearch.client:transport:${version}": ':client:transport',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
-    "org.elasticsearch.distribution.zip:elasticsearch:${bwcVersion}": ':distribution:bwc-zip',
+    "org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}": ':distribution:bwc-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
     "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
     "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',

distribution/bwc-zip/build.gradle

@@ -21,70 +21,74 @@ import java.util.regex.Matcher
 import org.elasticsearch.gradle.LoggedExec
 
 /**
- * This is a dummy project which does a local worktree checkout of the previous
- * major version's stable branch, and builds a snapshot. This allows backcompat
- * tests in the next major version to test against the next unreleased minor
- * version, without relying on snapshots.
+ * This is a dummy project which does a local checkout of the previous
+ * wire compat version's branch, and builds a snapshot. This allows backcompat
+ * tests to test against the next unreleased version, closest to this version,
+ * without relying on snapshots.
  */
 
-apply plugin: 'distribution'
+String bwcVersion = wireCompatVersions[-1]
+if (bwcVersion.endsWith('-SNAPSHOT')) {
+  apply plugin: 'distribution'
 
-File checkoutDir = file("${buildDir}/bwc/checkout-5.x")
-task createClone(type: LoggedExec) {
-  onlyIf { checkoutDir.exists() == false }
-  commandLine = ['git', 'clone', rootDir, checkoutDir]
-}
+  def (String major, String minor, String bugfix) = bwcVersion.split('\\.')
+  String bwcBranch = bugfix == '0-SNAPSHOT' ? "${major}.x" : "${major}.${minor}"
+  File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
+  task createClone(type: LoggedExec) {
+    onlyIf { checkoutDir.exists() == false }
+    commandLine = ['git', 'clone', rootDir, checkoutDir]
+  }
 
-// we use regular Exec here to ensure we always get output, regardless of logging level
-task findUpstream(type: Exec) {
-  dependsOn createClone
-  workingDir = checkoutDir
-  commandLine = ['git', 'remote', '-v']
-  ignoreExitValue = true
-  ByteArrayOutputStream output = new ByteArrayOutputStream()
-  standardOutput = output
-  doLast {
-    if (execResult.exitValue != 0) {
-      output.toString('UTF-8').eachLine { line -> logger.error(line) }
-      execResult.assertNormalExitValue()
-    }
-    project.ext.upstreamExists = false
-    output.toString('UTF-8').eachLine {
-      if (it.contains("upstream")) {
-        project.ext.upstreamExists = true
+  // we use regular Exec here to ensure we always get output, regardless of logging level
+  task findUpstream(type: Exec) {
+    dependsOn createClone
+    workingDir = checkoutDir
+    commandLine = ['git', 'remote', '-v']
+    ignoreExitValue = true
+    ByteArrayOutputStream output = new ByteArrayOutputStream()
+    standardOutput = output
+    doLast {
+      if (execResult.exitValue != 0) {
+        output.toString('UTF-8').eachLine { line -> logger.error(line) }
+        execResult.assertNormalExitValue()
+      }
+      project.ext.upstreamExists = false
+      output.toString('UTF-8').eachLine {
+        if (it.contains("upstream")) {
+          project.ext.upstreamExists = true
+        }
       }
     }
   }
-}
 
-task addUpstream(type: LoggedExec) {
-  dependsOn findUpstream
-  onlyIf { project.ext.upstreamExists == false }
-  workingDir = checkoutDir
-  commandLine = ['git', 'remote', 'add', 'upstream', 'https://github.com/elastic/elasticsearch.git']
-}
+  task addUpstream(type: LoggedExec) {
+    dependsOn findUpstream
+    onlyIf { project.ext.upstreamExists == false }
+    workingDir = checkoutDir
+    commandLine = ['git', 'remote', 'add', 'upstream', 'https://github.com/elastic/elasticsearch.git']
+  }
 
-task fetchLatest(type: LoggedExec) {
-  onlyIf { project.gradle.startParameter.isOffline() == false }
-  dependsOn addUpstream
-  workingDir = checkoutDir
-  commandLine = ['git', 'fetch', 'upstream']
-}
+  task fetchLatest(type: LoggedExec) {
+    onlyIf { project.gradle.startParameter.isOffline() == false }
+    dependsOn addUpstream
+    workingDir = checkoutDir
+    commandLine = ['git', 'fetch', 'upstream']
+  }
 
-task checkoutBwcBranch(type: LoggedExec) {
-  dependsOn fetchLatest
-  workingDir = checkoutDir
-  commandLine = ['git', 'checkout', 'upstream/5.x']
-}
+  task checkoutBwcBranch(type: LoggedExec) {
+    dependsOn fetchLatest
+    workingDir = checkoutDir
+    commandLine = ['git', 'checkout', "upstream/${bwcBranch}"]
+  }
 
-File bwcZip = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${bwcVersion}.zip")
-task buildBwcVersion(type: GradleBuild) {
-  dependsOn checkoutBwcBranch
-  dir = checkoutDir
-  tasks = [':distribution:zip:assemble']
-}
+  File bwcZip = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${bwcVersion}.zip")
+  task buildBwcVersion(type: GradleBuild) {
+    dependsOn checkoutBwcBranch
+    dir = checkoutDir
+    tasks = [':distribution:zip:assemble']
+  }
 
-artifacts {
-  'default' file: bwcZip, name: 'elasticsearch', type: 'zip', builtBy: buildBwcVersion
+  artifacts {
+    'default' file: bwcZip, name: 'elasticsearch', type: 'zip', builtBy: buildBwcVersion
+  }
 }
-

qa/backwards-5.0/build.gradle

@@ -28,7 +28,7 @@ integTest {
 integTestCluster {
   numNodes = 4
   numBwcNodes = 2
-  bwcVersion = project.bwcVersion
+  bwcVersion = project.wireCompatVersions[-1]
   setting 'logger.org.elasticsearch', 'DEBUG'
 }
 

qa/rolling-upgrade/build.gradle

@@ -27,7 +27,7 @@ task oldClusterTest(type: RestIntegTestTask) {
 
 oldClusterTestCluster {
   distribution = 'zip'
-  bwcVersion = project.bwcVersion // TODO: either randomize, or make this settable with sysprop
+  bwcVersion = project.wireCompatVersions[-1] // TODO: either randomize, or make this settable with sysprop
   numBwcNodes = 2
   numNodes = 2
   clusterName = 'rolling-upgrade'