From 89ba8996c6c735fa6fa0bca1a5cf4268b5daf07e Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 21 Nov 2017 08:26:45 +0000 Subject: [PATCH] Consolidate version numbering semantics (#27397) Fixes to the build system, particularly around BWC testing, and to make future version bumps less painful. --- build.gradle | 120 +++-------- .../org/elasticsearch/gradle/Version.groovy | 55 ++++- .../gradle/VersionCollection.groovy | 193 ++++++++++++++++++ .../gradle/vagrant/VagrantTestPlugin.groovy | 3 +- distribution/bwc/build.gradle | 35 +--- qa/full-cluster-restart/build.gradle | 10 +- qa/mixed-cluster/build.gradle | 6 +- qa/query-builder-bwc/build.gradle | 6 +- qa/rolling-upgrade/build.gradle | 6 +- qa/verify-version-constants/build.gradle | 6 +- settings.gradle | 46 +++-- test/framework/build.gradle | 4 +- 12 files changed, 329 insertions(+), 161 deletions(-) create mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy diff --git a/build.gradle b/build.gradle index 08e9c946eaf..99404c15e3a 100644 --- a/build.gradle +++ b/build.gradle @@ -17,15 +17,15 @@ * under the License. */ -import java.nio.file.Path -import java.util.regex.Matcher -import org.eclipse.jgit.lib.Repository -import org.eclipse.jgit.lib.RepositoryBuilder -import org.gradle.plugins.ide.eclipse.model.SourceFolder + import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.BuildPlugin -import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionCollection +import org.elasticsearch.gradle.VersionProperties +import org.gradle.plugins.ide.eclipse.model.SourceFolder + +import java.nio.file.Path // common maven publishing configuration subprojects { @@ -67,72 +67,16 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) { } } -/* Introspect all versions of ES that may be tested agains for backwards +/* Introspect all versions of ES that may be tested against for backwards * compatibility. It is *super* important that this logic is the same as the * logic in VersionUtils.java, throwing out alphas because they don't have any * backwards compatibility guarantees and only keeping the latest beta or rc * in a branch if there are only betas and rcs in the branch so we have * *something* to test against. */ -Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT')) -int prevMajor = currentVersion.major - 1 -File versionFile = file('core/src/main/java/org/elasticsearch/Version.java') -List versionLines = versionFile.readLines('UTF-8') -List versions = [] -// keep track of the previous major version's last minor, so we know where wire compat begins -int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major -int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen -int prevBugfixIndex = -1 // index in the versions list of the last bugfix release from the prev major -for (String line : versionLines) { - /* Note that this skips alphas and betas which is fine because they aren't - * compatible with anything. */ - Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_beta\d+|_rc\d+)? .*/ - if (match.matches()) { - int major = Integer.parseInt(match.group(1)) - int minor = Integer.parseInt(match.group(2)) - int bugfix = Integer.parseInt(match.group(3)) - String suffix = (match.group(4) ?: '').replace('_', '-') - Version foundVersion = new Version(major, minor, bugfix, suffix, false) - if (currentVersion != foundVersion - && (major == prevMajor || major == currentVersion.major)) { - if (versions.isEmpty() || versions.last() != foundVersion) { - versions.add(foundVersion) - } else { - // Replace the earlier betas with later ones - Version last = versions.set(versions.size() - 1, foundVersion) - if (last.suffix == '') { - throw new InvalidUserDataException("Found two equal versions but" - + " the first one [$last] wasn't a beta.") - } - } - if (major == prevMajor && minor > lastPrevMinor) { - prevMinorIndex = versions.size() - 1 - lastPrevMinor = minor - } - } - if (major == prevMajor) { - prevBugfixIndex = versions.size() - 1 - } - } -} -if (versions.toSorted { it.id } != versions) { - println "Versions: ${versions}" - throw new GradleException("Versions.java contains out of order version constants") -} -if (prevBugfixIndex != -1) { - versions[prevBugfixIndex] = new Version(versions[prevBugfixIndex].major, versions[prevBugfixIndex].minor, - versions[prevBugfixIndex].bugfix, versions[prevBugfixIndex].suffix, true) -} -if (currentVersion.bugfix == 0) { - // If on a release branch, after the initial release of that branch, the bugfix version will - // be bumped, and will be != 0. On master and N.x branches, we want to test against the - // unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT, - // and the bwc distribution will checkout and build that version. - Version last = versions[-1] - versions[-1] = new Version(last.major, last.minor, last.bugfix, last.suffix, true) - if (last.bugfix == 0) { - versions[-2] = new Version( - versions[-2].major, versions[-2].minor, versions[-2].bugfix, versions[-2].suffix, true) - } +VersionCollection versions = new VersionCollection(file('core/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8')) +if (versions.currentVersion.toString() != VersionProperties.elasticsearch) { + throw new GradleException("The last version in Versions.java [${versions.currentVersion}] does not match " + + "VersionProperties.elasticsearch [${VersionProperties.elasticsearch}]") } // build metadata from previous build, contains eg hashes for bwc builds @@ -151,9 +95,10 @@ allprojects { // for ide hacks... isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse') isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea') - // for backcompat testing - indexCompatVersions = versions - wireCompatVersions = versions.subList(prevMinorIndex, versions.size()) + + // for BWC testing + versionCollection = versions + buildMetadata = buildMetadataMap } } @@ -171,13 +116,13 @@ task verifyVersions { Set knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) }) // Limit the known versions to those that should be index compatible, and are not future versions - knownVersions = knownVersions.findAll { it.major >= prevMajor && it.before(VersionProperties.elasticsearch) } + knownVersions = knownVersions.findAll { it.major >= versions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) } /* Limit the listed versions to those that have been marked as released. * Versions not marked as released don't get the same testing and we want * to make sure that we flip all unreleased versions to released as soon * as possible after release. */ - Set actualVersions = new TreeSet<>(indexCompatVersions.findAll { false == it.snapshot }) + Set actualVersions = new TreeSet<>(versions.versionsIndexCompatibleWithCurrent.findAll { false == it.snapshot }) // Finally, compare! if (knownVersions.equals(actualVersions) == false) { @@ -252,30 +197,17 @@ subprojects { "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats', "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator', ] - if (indexCompatVersions[-1].snapshot) { - /* The last and second to last versions can be snapshots. Rather than use - * snapshots built by CI we connect these versions to projects that build - * those those versions from the HEAD of the appropriate branch. */ - if (indexCompatVersions[-1].bugfix == 0) { - ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot' - ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot' - ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-stable-snapshot' - if (indexCompatVersions.size() > 1) { - ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot' - ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot' - ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot' - } - } else { - ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot' - ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot' - ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-1]}"] = ':distribution:bwc-release-snapshot' + + for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) { + if (version.branch != null) { + final String snapshotProject = ":distribution:bwc-snapshot-${version.branch}" + project(snapshotProject).ext.bwcVersion = version + ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${version}"] = snapshotProject + ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${version}"] = snapshotProject + ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${version}"] = snapshotProject } - } else if (indexCompatVersions[-2].snapshot) { - /* This is a terrible hack for the bump to 6.0.1 which will be fixed by #27397 */ - ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot' - ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot' - ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${indexCompatVersions[-2]}"] = ':distribution:bwc-release-snapshot' } + project.afterEvaluate { configurations.all { resolutionStrategy.dependencySubstitution { DependencySubstitutions subs -> diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy index ee28dd8a994..7dbd943b62d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy @@ -31,9 +31,10 @@ public class Version { final int major final int minor - final int bugfix + final int revision final int id final boolean snapshot + final String branch /** * Suffix on the version name. Unlike Version.java the build does not * consider alphas and betas different versions, it just preserves the @@ -41,14 +42,15 @@ public class Version { */ final String suffix - public Version(int major, int minor, int bugfix, - String suffix, boolean snapshot) { + public Version(int major, int minor, int revision, + String suffix, boolean snapshot, String branch) { this.major = major this.minor = minor - this.bugfix = bugfix + this.revision = revision this.snapshot = snapshot this.suffix = suffix - this.id = major * 100000 + minor * 1000 + bugfix * 10 + + this.branch = branch + this.id = major * 100000 + minor * 1000 + revision * 10 + (snapshot ? 1 : 0) } @@ -58,13 +60,13 @@ public class Version { throw new InvalidUserDataException("Invalid version [${s}]") } return new Version(m.group(1) as int, m.group(2) as int, - m.group(3) as int, m.group(4) ?: '', m.group(5) != null) + m.group(3) as int, m.group(4) ?: '', m.group(5) != null, null) } @Override public String toString() { String snapshotStr = snapshot ? '-SNAPSHOT' : '' - return "${major}.${minor}.${bugfix}${suffix}${snapshotStr}" + return "${major}.${minor}.${revision}${suffix}${snapshotStr}" } public boolean before(String compareTo) { @@ -82,4 +84,43 @@ public class Version { public boolean after(String compareTo) { return id > fromString(compareTo).id } + + public boolean onOrBeforeIncludingSuffix(Version otherVersion) { + if (id != otherVersion.id) { + return id < otherVersion.id + } + + if (suffix == '') { + return otherVersion.suffix == '' + } + + return otherVersion.suffix == '' || suffix < otherVersion.suffix + } + + boolean equals(o) { + if (this.is(o)) return true + if (getClass() != o.class) return false + + Version version = (Version) o + + if (id != version.id) return false + if (major != version.major) return false + if (minor != version.minor) return false + if (revision != version.revision) return false + if (snapshot != version.snapshot) return false + if (suffix != version.suffix) return false + + return true + } + + int hashCode() { + int result + result = major + result = 31 * result + minor + result = 31 * result + revision + result = 31 * result + id + result = 31 * result + (snapshot ? 1 : 0) + result = 31 * result + (suffix != null ? suffix.hashCode() : 0) + return result + } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy new file mode 100644 index 00000000000..7ba0dc19e49 --- /dev/null +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy @@ -0,0 +1,193 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle + +import org.gradle.api.GradleException + +import java.util.regex.Matcher + +/** + * The collection of version constants declared in Version.java, for use in BWC testing. + */ +class VersionCollection { + + private final List versions + + /** + * Construct a VersionCollection from the lines of the Version.java file. + * @param versionLines The lines of the Version.java file. + */ + VersionCollection(List versionLines) { + + List versions = [] + + for (final String line : versionLines) { + final Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_alpha\d+|_beta\d+|_rc\d+)? .*/ + if (match.matches()) { + final Version foundVersion = new Version( + Integer.parseInt(match.group(1)), Integer.parseInt(match.group(2)), + Integer.parseInt(match.group(3)), (match.group(4) ?: '').replace('_', '-'), false, null) + + if (versions.size() > 0 && foundVersion.onOrBeforeIncludingSuffix(versions[-1])) { + throw new GradleException("Versions.java contains out of order version constants:" + + " ${foundVersion} should come before ${versions[-1]}") + } + + // Only keep the last alpha/beta/rc in the series + if (versions.size() > 0 && versions[-1].id == foundVersion.id) { + versions[-1] = foundVersion + } else { + versions.add(foundVersion) + } + } + } + + if (versions.empty) { + throw new GradleException("Unexpectedly found no version constants in Versions.java"); + } + + // The tip of each minor series (>= 5.6) is unreleased, so set their 'snapshot' flags + Version prevConsideredVersion = null + boolean found6xSnapshot = false + for (final int versionIndex = versions.size() - 1; versionIndex >= 0; versionIndex--) { + final Version currConsideredVersion = versions[versionIndex] + + if (prevConsideredVersion == null + || currConsideredVersion.major != prevConsideredVersion.major + || currConsideredVersion.minor != prevConsideredVersion.minor) { + + // This is a snapshot version. Work out its branch. NB this doesn't name the current branch correctly, but this doesn't + // matter as we don't BWC test against it. + String branch = "${currConsideredVersion.major}.${currConsideredVersion.minor}" + + if (false == found6xSnapshot && currConsideredVersion.major == 6) { + // TODO needs generalising to deal with when 7.x is cut, and when 6.x is deleted, and so on... + branch = "6.x" + found6xSnapshot = true + } + + versions[versionIndex] = new Version( + currConsideredVersion.major, currConsideredVersion.minor, + currConsideredVersion.revision, currConsideredVersion.suffix, true, branch) + } + + if (currConsideredVersion.onOrBefore("5.6.0")) { + break + } + + prevConsideredVersion = currConsideredVersion + } + + this.versions = Collections.unmodifiableList(versions) + } + + /** + * @return The list of versions read from the Version.java file + */ + List getVersions() { + return Collections.unmodifiableList(versions) + } + + /** + * @return The latest version in the Version.java file, which must be the current version of the system. + */ + Version getCurrentVersion() { + return versions[-1] + } + + /** + * @return The snapshot at the end of the previous minor series in the current major series, or null if this is the first minor series. + */ + Version getBWCSnapshotForCurrentMajor() { + return getLastSnapshotWithMajor(currentVersion.major) + } + + /** + * @return The snapshot at the end of the previous major series, which must not be null. + */ + Version getBWCSnapshotForPreviousMajor() { + Version version = getLastSnapshotWithMajor(currentVersion.major - 1) + assert version != null : "getBWCSnapshotForPreviousMajor(): found no versions in the previous major" + return version + } + + /** + * @return The snapshot at the end of the previous-but-one minor series in the current major series, if the previous minor series + * exists and has not yet been released. Otherwise null. + */ + Version getBWCSnapshotForPreviousMinorOfCurrentMajor() { + // If we are at 6.2.0 but 6.1.0 has not yet been released then we + // need to test against 6.0.1-SNAPSHOT too + final Version v = BWCSnapshotForCurrentMajor + if (v == null || v.revision != 0 || v.minor == 0) { + return null + } + return versions.find { it.major == v.major && it.minor == v.minor - 1 && it.snapshot } + } + + private Version getLastSnapshotWithMajor(int targetMajor) { + final String currentVersion = currentVersion.toString() + final int snapshotIndex = versions.findLastIndexOf { + it.major == targetMajor && it.before(currentVersion) && it.snapshot + } + return snapshotIndex == -1 ? null : versions[snapshotIndex] + } + + private List versionsOnOrAfterExceptCurrent(Version minVersion) { + final String minVersionString = minVersion.toString() + return Collections.unmodifiableList(versions.findAll { + it.onOrAfter(minVersionString) && it != currentVersion + }) + } + + /** + * @return All earlier versions that should be tested for index BWC with the current version. + */ + List getVersionsIndexCompatibleWithCurrent() { + final Version firstVersionOfCurrentMajor = versions.find { it.major >= currentVersion.major - 1 } + return versionsOnOrAfterExceptCurrent(firstVersionOfCurrentMajor) + } + + private Version getMinimumWireCompatibilityVersion() { + final int firstIndexOfThisMajor = versions.findIndexOf { it.major == currentVersion.major } + if (firstIndexOfThisMajor == 0) { + return versions[0] + } + final Version lastVersionOfEarlierMajor = versions[firstIndexOfThisMajor - 1] + return versions.find { it.major == lastVersionOfEarlierMajor.major && it.minor == lastVersionOfEarlierMajor.minor } + } + + /** + * @return All earlier versions that should be tested for wire BWC with the current version. + */ + List getVersionsWireCompatibleWithCurrent() { + return versionsOnOrAfterExceptCurrent(minimumWireCompatibilityVersion) + } + + /** + * `gradle check` does not run all BWC tests. This defines which tests it does run. + * @return Versions to test for BWC during gradle check. + */ + List getBasicIntegrationTestVersions() { + // TODO these are the versions checked by `gradle check` for BWC tests. Their choice seems a litle arbitrary. + List result = [BWCSnapshotForPreviousMajor, BWCSnapshotForCurrentMajor] + return Collections.unmodifiableList(result.findAll { it != null }) + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 59a7359a639..2510620ac71 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -107,7 +107,8 @@ class VagrantTestPlugin implements Plugin { if (upgradeFromVersion == null) { String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0) final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16) - upgradeFromVersion = project.indexCompatVersions[new Random(seed).nextInt(project.indexCompatVersions.size())] + final def indexCompatVersions = project.versionCollection.versionsIndexCompatibleWithCurrent + upgradeFromVersion = indexCompatVersions[new Random(seed).nextInt(indexCompatVersions.size())] } DISTRIBUTION_ARCHIVES.each { diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index f4cc2628df9..49d320ef676 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -19,6 +19,8 @@ import org.elasticsearch.gradle.LoggedExec +import org.elasticsearch.gradle.Version +import java.util.regex.Matcher /** * This is a dummy project which does a local checkout of the previous @@ -26,41 +28,20 @@ import org.elasticsearch.gradle.LoggedExec * tests to test against the next unreleased version, closest to this version, * without relying on snapshots. */ -String bwcVersion -boolean enabled = true -if (project.name == 'bwc-stable-snapshot') { - /* bwc-stable is only used if the last version is on a stable branch instead - * of a bugfix branch */ - enabled = indexCompatVersions[-1].bugfix == 0 - bwcVersion = indexCompatVersions[-1] -} else if (project.name == 'bwc-release-snapshot') { - if (indexCompatVersions[-1].bugfix == 0) { - /* The last version is on a stable branch so it is handled by the bwc-stable - * project. This project will instead handle the version before that which - * *should* be on a stable branch. */ - bwcVersion = indexCompatVersions[-2] - } else { - // The last version is on a release branch so it is handled by this project - bwcVersion = indexCompatVersions[-1] - } -} else { +final Matcher match = project.name =~ /bwc-snapshot-(\d+\.(\d+|x))/ +if (!match.matches()) { throw new InvalidUserDataException("Unsupport project name ${project.name}") } +String bwcBranch = match.group(1) + +if (project.hasProperty('bwcVersion')) { + Version bwcVersion = project.ext.bwcVersion -if (enabled) { apply plugin: 'distribution' // Not published so no need to assemble tasks.remove(assemble) build.dependsOn.remove('assemble') - def (String major, String minor, String bugfix) = bwcVersion.split('\\.') - def (String currentMajor, String currentMinor, String currentBugfix) = version.split('\\.') - String bwcBranch - if (project.name == 'bwc-stable-snapshot' && major != currentMajor) { - bwcBranch = "${major}.x" - } else { - bwcBranch = "${major}.${minor}" - } File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}") final String remote = System.getProperty("tests.bwc.remote", "elastic") diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index f271dae5cfd..123b217a38a 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -17,8 +17,9 @@ * under the License. */ -import org.elasticsearch.gradle.test.RestIntegTestTask + import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.standalone-test' @@ -29,7 +30,7 @@ task bwcTest { group = 'verification' } -for (Version version : indexCompatVersions) { +for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) { String baseName = "v${version}" Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { @@ -104,9 +105,8 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr // basic integ tests includes testing bwc against the most recent version task integTest { if (project.bwc_tests_enabled) { - dependsOn "v${indexCompatVersions[-1]}#bwcTest" - if (indexCompatVersions[-1].bugfix == 0) { - dependsOn "v${indexCompatVersions[-2]}#bwcTest" + for (final def version : versionCollection.basicIntegrationTestVersions) { + dependsOn "v${version}#bwcTest" } } } diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 66cad0c6eb6..c5806b3546c 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -29,7 +29,7 @@ task bwcTest { group = 'verification' } -for (Version version : wireCompatVersions) { +for (Version version : versionCollection.versionsWireCompatibleWithCurrent) { String baseName = "v${version}" Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask) { @@ -68,8 +68,8 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr // basic integ tests includes testing bwc against the most recent version task integTest { - if (project.bwc_tests_enabled) { - dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"] + for (final def version : versionCollection.basicIntegrationTestVersions) { + dependsOn "v${version}#bwcTest" } } diff --git a/qa/query-builder-bwc/build.gradle b/qa/query-builder-bwc/build.gradle index f1e7ad6f640..16e9f6298fe 100644 --- a/qa/query-builder-bwc/build.gradle +++ b/qa/query-builder-bwc/build.gradle @@ -30,7 +30,7 @@ task bwcTest { group = 'verification' } -for (Version version : indexCompatVersions) { +for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) { String baseName = "v${version}" Task oldQueryBuilderTest = tasks.create(name: "${baseName}#oldQueryBuilderTest", type: RestIntegTestTask) { @@ -83,7 +83,9 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr // basic integ tests includes testing bwc against the most recent version task integTest { if (project.bwc_tests_enabled) { - dependsOn = ["v${indexCompatVersions[-1]}#bwcTest"] + for (final def version : versionCollection.basicIntegrationTestVersions) { + dependsOn "v${version}#bwcTest" + } } } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index fc3cf88b272..13cccbd0062 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -29,7 +29,7 @@ task bwcTest { group = 'verification' } -for (Version version : wireCompatVersions) { +for (Version version : versionCollection.versionsWireCompatibleWithCurrent) { String baseName = "v${version}" Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { @@ -111,7 +111,9 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr // basic integ tests includes testing bwc against the most recent version task integTest { if (project.bwc_tests_enabled) { - dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"] + for (final def version : versionCollection.basicIntegrationTestVersions) { + dependsOn "v${version}#bwcTest" + } } } diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle index b501ffe168b..111c4ccf20e 100644 --- a/qa/verify-version-constants/build.gradle +++ b/qa/verify-version-constants/build.gradle @@ -31,7 +31,7 @@ task bwcTest { group = 'verification' } -for (Version version : indexCompatVersions) { +for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) { String baseName = "v${version}" Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { mustRunAfter(precommit) @@ -57,7 +57,9 @@ for (Version version : indexCompatVersions) { test.enabled = false task integTest { - dependsOn = ["v${indexCompatVersions[-1]}#bwcTest"] + for (final def version : versionCollection.basicIntegrationTestVersions) { + dependsOn "v${version}#bwcTest" + } } task verifyDocsLuceneVersion { diff --git a/settings.gradle b/settings.gradle index 997f3b30da5..cdac374198b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,8 +16,6 @@ List projects = [ 'client:benchmark', 'benchmarks', 'distribution:integ-test-zip', - 'distribution:bwc-release-snapshot', - 'distribution:bwc-stable-snapshot', 'distribution:zip', 'distribution:tar', 'distribution:deb', @@ -97,6 +95,12 @@ for (File example : examplePluginsDir.listFiles()) { examplePlugins.add(example.name) } +/* Create projects for building BWC snapshot distributions from the heads of other branches */ +final List branches = ['5.6', '6.0', '6.1', '6.x'] +for (final String branch : branches) { + projects.add("distribution:bwc-snapshot-${branch}".toString()) +} + boolean isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse') if (isEclipse) { // eclipse cannot handle an intermediate dependency between main and test, so we must create separate projects @@ -113,12 +117,11 @@ for (String example : examplePlugins) { project(":example-plugins:${example}").projectDir = new File(rootProject.projectDir, "plugins/examples/${example}") } -/* bwc and bwc-unreleased share the same build directory and build file, but - * apply to different backwards compatibility branches. */ -project(':distribution:bwc-release-snapshot').projectDir = - new File(rootProject.projectDir, 'distribution/bwc') -project(':distribution:bwc-stable-snapshot').projectDir = - new File(rootProject.projectDir, 'distribution/bwc') +/* The BWC snapshot projects share the same build directory and build file, + * but apply to different backwards compatibility branches. */ +for (final String branch : branches) { + project(":distribution:bwc-snapshot-${branch}").projectDir = new File(rootProject.projectDir, 'distribution/bwc') +} if (isEclipse) { project(":core").projectDir = new File(rootProject.projectDir, 'core/src/main') @@ -133,18 +136,29 @@ if (isEclipse) { * of the dir hierarchy to have a build.gradle. Otherwise we would have to iterate * all files/directories in the source tree to find all projects. */ -void addSubProjects(String path, File dir) { +void addSubProjects(String path, File dir, List projects, List branches) { if (dir.isDirectory() == false) return; if (dir.name == 'buildSrc') return; if (new File(dir, 'build.gradle').exists() == false) return; - String projectName = "${path}:${dir.name}" + final String projectName = "${path}:${dir.name}" include projectName - if (path.isEmpty()) { - project(projectName).projectDir = dir - } - for (File subdir : dir.listFiles()) { - addSubProjects(projectName, subdir) + + if (dir.name == 'bwc-snapshot-dummy-projects') { + for (final String branch : branches) { + final String snapshotProjectName = "${projectName}:bwc-snapshot-${branch}" + projects.add(snapshotProjectName) + include snapshotProjectName + project("${snapshotProjectName}").projectDir = dir + } + // TODO do we want to assert that there's nothing else in the bwc directory? + } else { + if (path.isEmpty()) { + project(projectName).projectDir = dir + } + for (File subdir : dir.listFiles()) { + addSubProjects(projectName, subdir, projects, branches) + } } } @@ -152,6 +166,6 @@ void addSubProjects(String path, File dir) { File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra") if (extraProjects.exists()) { for (File extraProjectDir : extraProjects.listFiles()) { - addSubProjects('', extraProjectDir) + addSubProjects('', extraProjectDir, projects, branches) } } diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 558bb2c851c..bbdbab5b233 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -69,6 +69,6 @@ task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConven precommit.dependsOn namingConventionsMain test.configure { - systemProperty 'tests.gradle_index_compat_versions', indexCompatVersions.join(',') - systemProperty 'tests.gradle_wire_compat_versions', wireCompatVersions.join(',') + systemProperty 'tests.gradle_index_compat_versions', versionCollection.versionsIndexCompatibleWithCurrent.join(',') + systemProperty 'tests.gradle_wire_compat_versions', versionCollection.versionsWireCompatibleWithCurrent.join(',') }