Tests: Use local clone build of 5.x with bwc tests (#22946)

The current rest backcompat tests, which run against a mixed cluster of
5.x and 6.0 nodes, depend on snapshot builds of 5.x. However, this has
the potential for inconsistency that results in CI failures, and happens
quite often, whenever some backcompat logic is added to 5.x, but the bwc
test on master fails because the 5.x code has not yet been published as
a snapshot.

This change creates a git clone of the 5.x branch,
builds the zip distribution, and ties that into gradle substitutions for
the 5.x version.

build.gradle

@@ -112,6 +112,8 @@ subprojects {
     "org.elasticsearch.client:transport:${version}": ':client:transport',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
+    // TODO: don't use a hardcoded version here, introspect it from Version.java
+    "org.elasticsearch.distribution.zip:elasticsearch:5.4.0-SNAPSHOT": ':distribution:bwc-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
     "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
     "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
@@ -123,10 +125,12 @@ subprojects {
     "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
   ]
-  configurations.all {
-    resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
-      projectSubstitutions.each { k,v ->
-        subs.substitute(subs.module(k)).with(subs.project(v))
+  project.afterEvaluate {
+    configurations.all {
+      resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
+        projectSubstitutions.each { k,v ->
+          subs.substitute(subs.module(k)).with(subs.project(v))
+        }
       }
     }
   }

distribution/build.gradle

@@ -40,7 +40,8 @@ buildscript {
   }
 }
 
-Collection distributions = project.subprojects.findAll { it.path.contains(':tools') == false }
+Collection distributions = project.subprojects.findAll {
+   it.path.contains(':tools') == false && it.name != 'bwc-zip' }
 
 /*****************************************************************************
  *                                Notice file                                *
@@ -82,7 +83,7 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each
   }
   // We would like to make sure integ tests for the distribution run after
   // integ tests for the modules included in the distribution.
-  project.configure(distributions.findAll { it.name != 'integ-test-zip' }) { Project distribution ->
+  project.configure(distributions.findAll { it.name != 'integ-test-zip'}) { Project distribution ->
     distribution.afterEvaluate({
       // some integTest tasks will have multiple finalizers
       distribution.integTest.mustRunAfter module.tasks.find { t -> t.name.matches(".*integTest\$") }

distribution/bwc-zip/build.gradle

@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.regex.Matcher
+import org.elasticsearch.gradle.LoggedExec
+
+/**
+ * This is a dummy project which does a local worktree checkout of the previous
+ * major version's stable branch, and builds a snapshot. This allows backcompat
+ * tests in the next major version to test against the next unreleased minor
+ * version, without relying on snapshots.
+ */
+
+apply plugin: 'distribution'
+
+// TODO: generate this, by introspecting Version.java for last previous minor
+String BWC_VERSION = "5.4.0-SNAPSHOT"
+
+String checkoutDir = "${buildDir}/bwc/checkout-5.x"
+task createClone(type: LoggedExec) {
+  onlyIf { new File(checkoutDir).exists() == false }
+  commandLine = ['git', 'clone', rootDir, checkoutDir]
+}
+
+task findUpstream(type: LoggedExec) {
+  dependsOn createClone
+  workingDir = checkoutDir
+  commandLine = ['git', 'remote', '-v']
+  doLast {
+    project.ext.upstreamExists = false
+    output.toString('UTF-8').eachLine {
+      if (it.contains("remote \"upstream\"")) {
+        project.ext.upstreamExists = true
+      }
+    }
+  }
+}
+
+task addUpstream(type: LoggedExec) {
+  dependsOn findUpstream
+  onlyIf { project.ext.upstreamExists == false }
+  workingDir = checkoutDir
+  commandLine = ['git', 'remote', 'add', 'upstream', 'https://github.com/elastic/elasticsearch.git']
+}
+
+task fetchLatest(type: LoggedExec) {
+  dependsOn addUpstream
+  workingDir = checkoutDir
+  commandLine = ['git', 'fetch', 'upstream', '5.x']
+}
+
+task checkoutBwcBranch(type: LoggedExec) {
+  dependsOn fetchLatest
+  workingDir = checkoutDir
+  commandLine = ['git', 'checkout', 'upstream/5.x']
+}
+
+File bwcZip = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${BWC_VERSION}.zip")
+task buildBwcVersion(type: LoggedExec) {
+  dependsOn checkoutBwcBranch
+  workingDir = checkoutDir
+  commandLine = ['gradle', ':distribution:zip:assemble']
+}
+
+artifacts {
+  'default' file: bwcZip, name: 'elasticsearch', type: 'zip', builtBy: buildBwcVersion
+}
+

modules/lang-painless/build.gradle

@@ -36,6 +36,7 @@ dependencyLicenses {
 // regeneration logic, comes in via ant right now
 // don't port it to gradle, it works fine.
 
+/*
 configurations {
   regenerate
 }
@@ -44,8 +45,10 @@ dependencies {
   regenerate 'org.antlr:antlr4:4.5.1-1'
 }
 
+// TODO: this is causing a resolve at configurationt time. we really should not be using ant anymore
 ant.references['regenerate.classpath'] = new Path(ant.project, configurations.regenerate.asPath)
 ant.importBuild 'ant.xml'
+*/
 
 integTestCluster {
   setting 'script.max_compilations_per_minute', '1000'

qa/backwards-5.0/build.gradle

@@ -20,19 +20,7 @@
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-/* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor.
- * Since we don't have a version to test against we currently use the hardcoded snapshot for to basically run
- * against ourselves. To test that using a different version go to distribution/zip and execute:
- *   gradle clean publishToMavenLocal -Dbuild.snapshot=false
- *
- * This installs the release-build into a local .m2 repository, then change this version here to:
- *   bwcVersion = "5.0.0"
- *
- * now you can run the bwc tests with:
- *   gradle check -Drepos.mavenlocal=true
- *
- * (-Drepos.mavenlocal=true will force gradle to look for the zip distribution in the local .m2 repository)
- */
+/* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor.  */
 integTest {
   includePackaged = true
 }
@@ -44,8 +32,3 @@ integTestCluster {
   setting 'logger.org.elasticsearch', 'DEBUG'
 }
 
-repositories {
-  maven {
-    url "https://oss.sonatype.org/content/repositories/snapshots/"
-  }
-}

settings.gradle

@@ -15,6 +15,7 @@ List projects = [
   'client:benchmark',
   'benchmarks',
   'distribution:integ-test-zip',
+  'distribution:bwc-zip',
   'distribution:zip',
   'distribution:tar',
   'distribution:deb',