diff --git a/build.gradle b/build.gradle
index fd97470ec6c..987f924ea50 100644
--- a/build.gradle
+++ b/build.gradle
@@ -112,6 +112,8 @@ subprojects {
     "org.elasticsearch.client:transport:${version}": ':client:transport',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
+    // TODO: don't use a hardcoded version here, introspect it from Version.java
+    "org.elasticsearch.distribution.zip:elasticsearch:5.4.0-SNAPSHOT": ':distribution:bwc-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
     "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
     "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
@@ -123,10 +125,12 @@ subprojects {
     "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
   ]
-  configurations.all {
-    resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
-      projectSubstitutions.each { k,v ->
-        subs.substitute(subs.module(k)).with(subs.project(v))
+  project.afterEvaluate {
+    configurations.all {
+      resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
+        projectSubstitutions.each { k,v ->
+          subs.substitute(subs.module(k)).with(subs.project(v))
+        }
       }
     }
   }
diff --git a/distribution/build.gradle b/distribution/build.gradle
index e95ccf09320..2a8094eb800 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -40,7 +40,8 @@ buildscript {
   }
 }
 
-Collection distributions = project.subprojects.findAll { it.path.contains(':tools') == false }
+Collection distributions = project.subprojects.findAll {
+   it.path.contains(':tools') == false && it.name != 'bwc-zip' }
 
 /*****************************************************************************
  *                                Notice file                                *
@@ -82,7 +83,7 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each
   }
   // We would like to make sure integ tests for the distribution run after
   // integ tests for the modules included in the distribution.
-  project.configure(distributions.findAll { it.name != 'integ-test-zip' }) { Project distribution ->
+  project.configure(distributions.findAll { it.name != 'integ-test-zip'}) { Project distribution ->
     distribution.afterEvaluate({
       // some integTest tasks will have multiple finalizers
       distribution.integTest.mustRunAfter module.tasks.find { t -> t.name.matches(".*integTest\$") }
diff --git a/distribution/bwc-zip/build.gradle b/distribution/bwc-zip/build.gradle
new file mode 100644
index 00000000000..258391cd5c8
--- /dev/null
+++ b/distribution/bwc-zip/build.gradle
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.regex.Matcher
+import org.elasticsearch.gradle.LoggedExec
+
+/**
+ * This is a dummy project which does a local worktree checkout of the previous
+ * major version's stable branch, and builds a snapshot. This allows backcompat
+ * tests in the next major version to test against the next unreleased minor
+ * version, without relying on snapshots.
+ */
+
+apply plugin: 'distribution'
+
+// TODO: generate this, by introspecting Version.java for last previous minor
+String BWC_VERSION = "5.4.0-SNAPSHOT"
+
+String checkoutDir = "${buildDir}/bwc/checkout-5.x"
+task createClone(type: LoggedExec) {
+  onlyIf { new File(checkoutDir).exists() == false }
+  commandLine = ['git', 'clone', rootDir, checkoutDir]
+}
+
+task findUpstream(type: LoggedExec) {
+  dependsOn createClone
+  workingDir = checkoutDir
+  commandLine = ['git', 'remote', '-v']
+  doLast {
+    project.ext.upstreamExists = false
+    output.toString('UTF-8').eachLine {
+      if (it.contains("remote \"upstream\"")) {
+        project.ext.upstreamExists = true
+      }
+    }
+  }
+}
+
+task addUpstream(type: LoggedExec) {
+  dependsOn findUpstream
+  onlyIf { project.ext.upstreamExists == false }
+  workingDir = checkoutDir
+  commandLine = ['git', 'remote', 'add', 'upstream', 'https://github.com/elastic/elasticsearch.git']
+}
+
+task fetchLatest(type: LoggedExec) {
+  dependsOn addUpstream
+  workingDir = checkoutDir
+  commandLine = ['git', 'fetch', 'upstream', '5.x']
+}
+
+task checkoutBwcBranch(type: LoggedExec) {
+  dependsOn fetchLatest
+  workingDir = checkoutDir
+  commandLine = ['git', 'checkout', 'upstream/5.x']
+}
+
+File bwcZip = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${BWC_VERSION}.zip")
+task buildBwcVersion(type: LoggedExec) {
+  dependsOn checkoutBwcBranch
+  workingDir = checkoutDir
+  commandLine = ['gradle', ':distribution:zip:assemble']
+}
+
+artifacts {
+  'default' file: bwcZip, name: 'elasticsearch', type: 'zip', builtBy: buildBwcVersion
+}
+
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index c9e013d8c6d..943d953ea48 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -36,6 +36,7 @@ dependencyLicenses {
 // regeneration logic, comes in via ant right now
 // don't port it to gradle, it works fine.
 
+/*
 configurations {
   regenerate
 }
@@ -44,8 +45,10 @@ dependencies {
   regenerate 'org.antlr:antlr4:4.5.1-1'
 }
 
+// TODO: this is causing a resolve at configurationt time. we really should not be using ant anymore
 ant.references['regenerate.classpath'] = new Path(ant.project, configurations.regenerate.asPath)
 ant.importBuild 'ant.xml'
+*/
 
 integTestCluster {
   setting 'script.max_compilations_per_minute', '1000'
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
index 90dd2f2d8bb..1e6fa0453e9 100644
--- a/qa/backwards-5.0/build.gradle
+++ b/qa/backwards-5.0/build.gradle
@@ -20,19 +20,7 @@
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-/* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor.
- * Since we don't have a version to test against we currently use the hardcoded snapshot for to basically run
- * against ourselves. To test that using a different version go to distribution/zip and execute:
- *   gradle clean publishToMavenLocal -Dbuild.snapshot=false
- *
- * This installs the release-build into a local .m2 repository, then change this version here to:
- *   bwcVersion = "5.0.0"
- *
- * now you can run the bwc tests with:
- *   gradle check -Drepos.mavenlocal=true
- *
- * (-Drepos.mavenlocal=true will force gradle to look for the zip distribution in the local .m2 repository)
- */
+/* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor.  */
 integTest {
   includePackaged = true
 }
@@ -44,8 +32,3 @@ integTestCluster {
   setting 'logger.org.elasticsearch', 'DEBUG'
 }
 
-repositories {
-  maven {
-    url "https://oss.sonatype.org/content/repositories/snapshots/"
-  }
-}
diff --git a/settings.gradle b/settings.gradle
index c0996f86a04..8e6d3d80a0e 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -15,6 +15,7 @@ List projects = [
   'client:benchmark',
   'benchmarks',
   'distribution:integ-test-zip',
+  'distribution:bwc-zip',
   'distribution:zip',
   'distribution:tar',
   'distribution:deb',