From ecaf6ef0018d0e1e950df9f4dd418a64822b5764 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 31 Aug 2016 16:45:03 -0700
Subject: [PATCH 01/15] Add rolling upgrade test project

---
 .../gradle/test/ClusterConfiguration.groovy   | 28 ++++++++-------
 .../gradle/test/ClusterFormationTasks.groovy  |  4 +--
 .../elasticsearch/gradle/test/NodeInfo.groovy | 19 ++++++++--
 .../gradle/test/RestIntegTestTask.groovy      | 21 +++++++----
 qa/rolling-upgrade/build.gradle               | 36 +++++++++++++++++++
 settings.gradle                               |  1 +
 6 files changed, 85 insertions(+), 24 deletions(-)
 create mode 100644 qa/rolling-upgrade/build.gradle

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index 6a2375efc62..3b34b45df0d 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -47,6 +47,14 @@ class ClusterConfiguration {
     @Input
     int transportPort = 0
 
+    /** An override of the data directory. This may only be used with a single node. */
+    @Input
+    File dataDir = null
+
+    /** Optional override of the cluster name. */
+    @Input
+    String clusterName = null
+
     @Input
     boolean daemonize = true
 
@@ -59,13 +67,15 @@ class ClusterConfiguration {
             " " + System.getProperty('tests.jvm.argline', '')
 
     /**
-     * The seed nodes port file. In the case the cluster has more than one node we use a seed node
-     * to form the cluster. The file is null if there is no seed node yet available.
+     * A uri that should be used for the unicast host list.
      *
-     * Note: this can only be null if the cluster has only one node or if the first node is not yet
-     * configured. All nodes but the first node should see a non null value.
+     * This allows multi node clusters, or a new cluster to connect to an existing cluster.
+     * The type is Object to allow lazy evaluation. Typically this would be set with a
+     * closure in a GString like:
+     *
+     * {@code "$&#123;-> node.transportUri()&#125;"}
      */
-    File seedNodePortsFile
+    Object unicastTransportUri = null
 
     /**
      * A closure to call before the cluster is considered ready. The closure is passed the node info,
@@ -137,12 +147,4 @@ class ClusterConfiguration {
         }
         extraConfigFiles.put(path, sourceFile)
     }
-
-    /** Returns an address and port suitable for a uri to connect to this clusters seed node over transport protocol*/
-    String seedNodeTransportUri() {
-        if (seedNodePortsFile != null) {
-            return seedNodePortsFile.readLines("UTF-8").get(0)
-        }
-        return null;
-    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 8819c63080a..5acf7fde55d 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -48,7 +48,7 @@ class ClusterFormationTasks {
      *
      * Returns a NodeInfo object for the first node in the cluster.
      */
-    static NodeInfo setup(Project project, Task task, ClusterConfiguration config) {
+    static List<NodeInfo> setup(Project project, Task task, ClusterConfiguration config) {
         if (task.getEnabled() == false) {
             // no need to add cluster formation tasks if the task won't run!
             return
@@ -110,7 +110,7 @@ class ClusterFormationTasks {
         task.dependsOn(wait)
 
         // delay the resolution of the uri by wrapping in a closure, so it is not used until read for tests
-        return nodes[0]
+        return nodes
     }
 
     /** Adds a dependency on the given distribution */
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index 5d9961a0425..e1834e26fb1 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -57,6 +57,9 @@ class NodeInfo {
     /** config directory */
     File confDir
 
+    /** data directory */
+    File dataDir
+
     /** THE config file */
     File configFile
 
@@ -95,11 +98,23 @@ class NodeInfo {
         this.config = config
         this.nodeNum = nodeNum
         this.sharedDir = sharedDir
-        clusterName = "${task.path.replace(':', '_').substring(1)}"
+        if (config.clusterName != null) {
+            clusterName = config.clusterName
+        } else {
+            clusterName = "${task.path.replace(':', '_').substring(1)}"
+        }
         baseDir = new File(project.buildDir, "cluster/${task.name} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
         homeDir = homeDir(baseDir, config.distribution, nodeVersion)
         confDir = confDir(baseDir, config.distribution, nodeVersion)
+        if (config.dataDir != null) {
+            if (config.numNodes != 1) {
+                throw new IllegalArgumentException("Cannot set data dir for integ test with more than one node")
+            }
+            dataDir = config.dataDir
+        } else {
+            dataDir = new File(homeDir, "data")
+        }
         configFile = new File(confDir, 'elasticsearch.yml')
         // even for rpm/deb, the logs are under home because we dont start with real services
         File logsDir = new File(homeDir, 'logs')
@@ -140,7 +155,7 @@ class NodeInfo {
             }
         }
         env.put('ES_JVM_OPTIONS', new File(confDir, 'jvm.options'))
-        args.addAll("-E", "path.conf=${confDir}")
+        args.addAll("-E", "path.conf=${confDir}", "-E", "path.data=${dataDir}")
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {
             args.add('"') // end the entire command, quoted
         }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
index c6463d28811..d50937408e7 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
@@ -34,6 +34,9 @@ public class RestIntegTestTask extends RandomizedTestingTask {
 
     ClusterConfiguration clusterConfig
 
+    /** Info about nodes in the integ test cluster. Note this is *not* available until runtime. */
+    List<NodeInfo> nodes
+
     /** Flag indicating whether the rest tests in the rest spec should be run. */
     @Input
     boolean includePackaged = false
@@ -52,6 +55,12 @@ public class RestIntegTestTask extends RandomizedTestingTask {
         parallelism = '1'
         include('**/*IT.class')
         systemProperty('tests.rest.load_packaged', 'false')
+        systemProperty('tests.rest.cluster', "${-> nodes[0].httpUri()}")
+        systemProperty('tests.config.dir', "${-> nodes[0].confDir}")
+        // TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
+        // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
+        // both as separate sysprops
+        systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")
 
         // copy the rest spec/tests into the test resources
         RestSpecHack.configureDependencies(project)
@@ -61,13 +70,7 @@ public class RestIntegTestTask extends RandomizedTestingTask {
         // this must run after all projects have been configured, so we know any project
         // references can be accessed as a fully configured
         project.gradle.projectsEvaluated {
-            NodeInfo node = ClusterFormationTasks.setup(project, this, clusterConfig)
-            systemProperty('tests.rest.cluster', "${-> node.httpUri()}")
-            systemProperty('tests.config.dir', "${-> node.confDir}")
-            // TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
-            // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
-            // both as separate sysprops
-            systemProperty('tests.cluster', "${-> node.transportUri()}")
+            nodes = ClusterFormationTasks.setup(project, this, clusterConfig)
         }
     }
 
@@ -88,6 +91,10 @@ public class RestIntegTestTask extends RandomizedTestingTask {
         return clusterConfig
     }
 
+    public List<NodeInfo> getNodes() {
+        return nodes
+    }
+
     @Override
     public Task dependsOn(Object... dependencies) {
         super.dependsOn(dependencies)
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
new file mode 100644
index 00000000000..d15a76daa92
--- /dev/null
+++ b/qa/rolling-upgrade/build.gradle
@@ -0,0 +1,36 @@
+
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+task oldClusterTest(type: RestIntegTestTask) {
+  mustRunAfter(precommit)
+  cluster {
+    distribution = 'zip'
+    bwcVersion = '2.4.0' // TODO: either randomize, or make this settable with sysprop
+    numBwcNodes = 2
+    numNodes = 0
+    clusterName = 'rolling-upgrade'
+  }
+  systemProperty 'tests.rest.suite', 'old_cluster'
+}
+
+task mixedClusterTest(type: RestIntegTestTask) {
+  dependsOn(oldClusterTest, 'oldClusterTest.node1
+  cluster {
+    distribution = 'zip'
+    clusterName = 'rolling-upgrade'
+  }
+  systemProperty 'tests.rest.suite', 'mixed_cluster'
+}
+
+task upgradedClusterTest(type: RestIntegTestTask) {
+  cluster {
+    distribution = 'zip'
+    clusterName = 'rolling-upgrade'
+  }
+  systemProperty 'tests.rest.suite', 'upgraded_cluster'
+}
+
+task integTest {
+  dependsOn = [upgradedClusterTest]
+}
+check.dependsOn(integTest)
diff --git a/settings.gradle b/settings.gradle
index 904fb69469d..b2ddae925a3 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -55,6 +55,7 @@ List projects = [
   'plugins:store-smb',
   'qa:backwards-5.0',
   'qa:evil-tests',
+  'qa:rolling-upgrade',
   'qa:smoke-test-client',
   'qa:smoke-test-ingest-with-all-dependencies',
   'qa:smoke-test-ingest-disabled',

From 5d8aa6b4fecbe92a6aca9175b0022a8ded721b8f Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 2 Sep 2016 11:10:17 -0400
Subject: [PATCH 02/15] Adds tests for rolling upgrades to execute

---
 .../gradle/test/ClusterConfiguration.groovy   |  5 +--
 .../gradle/test/ClusterFormationTasks.groovy  | 30 +++++++------
 qa/rolling-upgrade/build.gradle               | 34 ++++++++++++++-
 .../UpgradeClusterClientYamlTestSuiteIT.java  | 43 +++++++++++++++++++
 .../mixed_cluster/10_basic.yaml               | 22 ++++++++++
 .../old_cluster/10_basic.yaml                 | 22 ++++++++++
 .../upgraded_cluster/10_basic.yaml            | 22 ++++++++++
 7 files changed, 159 insertions(+), 19 deletions(-)
 create mode 100644 qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
 create mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec.test/mixed_cluster/10_basic.yaml
 create mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec.test/old_cluster/10_basic.yaml
 create mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec.test/upgraded_cluster/10_basic.yaml

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index 3b34b45df0d..689b7ed9ef4 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -20,8 +20,6 @@ package org.elasticsearch.gradle.test
 
 import org.gradle.api.GradleException
 import org.gradle.api.Project
-import org.gradle.api.artifacts.Configuration
-import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 
 /** Configuration for an elasticsearch cluster, used for integration tests. */
@@ -75,7 +73,8 @@ class ClusterConfiguration {
      *
      * {@code "$&#123;-> node.transportUri()&#125;"}
      */
-    Object unicastTransportUri = null
+    @Input
+    Closure unicastTransportUri = null
 
     /**
      * A closure to call before the cluster is considered ready. The closure is passed the node info,
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 5acf7fde55d..58b87e5c2d5 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -86,6 +86,7 @@ class ClusterFormationTasks {
             configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchBwcDistro, config.bwcVersion)
         }
 
+        NodeInfo seedNode = null
         for (int i = 0; i < config.numNodes; ++i) {
             // we start N nodes and out of these N nodes there might be M bwc nodes.
             // for each of those nodes we might have a different configuratioon
@@ -95,15 +96,11 @@ class ClusterFormationTasks {
                 distro = project.configurations.elasticsearchBwcDistro
             }
             NodeInfo node = new NodeInfo(config, i, project, task, elasticsearchVersion, sharedDir)
-            if (i == 0) {
-                if (config.seedNodePortsFile != null) {
-                    // we might allow this in the future to be set but for now we are the only authority to set this!
-                    throw new GradleException("seedNodePortsFile has a non-null value but first node has not been intialized")
-                }
-                config.seedNodePortsFile = node.transportPortsFile;
-            }
             nodes.add(node)
-            startTasks.add(configureNode(project, task, cleanup, node, distro))
+            if (i == 0) {
+                seedNode = node
+            }
+            startTasks.add(configureNode(project, task, cleanup, node, distro, seedNode))
         }
 
         Task wait = configureWaitTask("${task.name}#wait", project, nodes, startTasks)
@@ -141,7 +138,7 @@ class ClusterFormationTasks {
      *
      * @return a task which starts the node.
      */
-    static Task configureNode(Project project, Task task, Object dependsOn, NodeInfo node, Configuration configuration) {
+    static Task configureNode(Project project, Task task, Object dependsOn, NodeInfo node, Configuration configuration, NodeInfo seedNode) {
 
         // tasks are chained so their execution order is maintained
         Task setup = project.tasks.create(name: taskName(task, node, 'clean'), type: Delete, dependsOn: dependsOn) {
@@ -154,7 +151,7 @@ class ClusterFormationTasks {
         setup = configureCheckPreviousTask(taskName(task, node, 'checkPrevious'), project, setup, node)
         setup = configureStopTask(taskName(task, node, 'stopPrevious'), project, setup, node)
         setup = configureExtractTask(taskName(task, node, 'extract'), project, setup, node, configuration)
-        setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node)
+        setup = configureWriteConfigTask(taskName(task, node, 'configure'), project, setup, node, seedNode)
         setup = configureExtraConfigFilesTask(taskName(task, node, 'extraConfig'), project, setup, node)
         setup = configureCopyPluginsTask(taskName(task, node, 'copyPlugins'), project, setup, node)
 
@@ -249,7 +246,7 @@ class ClusterFormationTasks {
     }
 
     /** Adds a task to write elasticsearch.yml for the given node configuration */
-    static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node) {
+    static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node, NodeInfo seedNode) {
         Map esConfig = [
                 'cluster.name'                 : node.clusterName,
                 'pidfile'                      : node.pidFile,
@@ -266,15 +263,20 @@ class ClusterFormationTasks {
 
         Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
         writeConfig.doFirst {
-            if (node.nodeNum > 0) { // multi-node cluster case, we have to wait for the seed node to startup
+            if (node.config.unicastTransportUri != null) {
+                // if the unicast transport uri was specified, use it for all nodes
+                // this will typically be the case if all the nodes we are setting up
+                // should connect to a master in an already formed cluster
+                esConfig['discovery.zen.ping.unicast.hosts'] = node.config.unicastTransportUri()
+            } else if (node.nodeNum > 0) { // multi-node cluster case, we have to wait for the seed node to startup
                 ant.waitfor(maxwait: '20', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
                     resourceexists {
-                        file(file: node.config.seedNodePortsFile.toString())
+                        file(file: seedNode.transportPortsFile.toString())
                     }
                 }
                 // the seed node is enough to form the cluster - all subsequent nodes will get the seed node as a unicast
                 // host and join the cluster via that.
-                esConfig['discovery.zen.ping.unicast.hosts'] = "\"${node.config.seedNodeTransportUri()}\""
+                esConfig['discovery.zen.ping.unicast.hosts'] = "\"${seedNode.transportUri()}\""
             }
             File configFile = new File(node.confDir, 'elasticsearch.yml')
             logger.info("Configuring ${configFile}")
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index d15a76daa92..8fb53093585 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -1,7 +1,27 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
+
+import org.elasticsearch.gradle.test.NodeInfo
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
-task oldClusterTest(type: RestIntegTestTask) {
+RestIntegTestTask oldClusterTask = task oldClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
   cluster {
     distribution = 'zip'
@@ -14,18 +34,28 @@ task oldClusterTest(type: RestIntegTestTask) {
 }
 
 task mixedClusterTest(type: RestIntegTestTask) {
-  dependsOn(oldClusterTest, 'oldClusterTest.node1
+  dependsOn(oldClusterTest, 'oldClusterTest.node1') // TODO: what does this `oldClusterTest.node1` do?  is it a task?
+  NodeInfo aliveNode = oldClusterTask.getNodes().get(0)
+  NodeInfo stoppedNode = oldClusterTask.getNodes().get(1)
   cluster {
     distribution = 'zip'
     clusterName = 'rolling-upgrade'
+    unicastTransportUri = { -> aliveNode.transportUri() }
+    dataDir = stoppedNode.dataDir
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
+
 }
 
 task upgradedClusterTest(type: RestIntegTestTask) {
+  // stop alive node from oldClusterTest and get its dataDir, and get alive node for unicast host
+  NodeInfo stoppedNode = null
+  NodeInfo aliveNode = null
   cluster {
     distribution = 'zip'
     clusterName = 'rolling-upgrade'
+    unicastTransportUri = { -> aliveNode.transportUri() }
+    dataDir = stoppedNode.dataDir
   }
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
 }
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
new file mode 100644
index 00000000000..f5cc3367202
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
+
+import java.io.IOException;
+
+@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
+public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+        return createParameters(0, 1);
+    }
+}
+
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/mixed_cluster/10_basic.yaml
new file mode 100644
index 00000000000..a46d7aaf044
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/mixed_cluster/10_basic.yaml
@@ -0,0 +1,22 @@
+---
+"Index data and search on the mixed cluster":
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v1_mixed", "f2": 5}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v2_mixed", "f2": 6}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v3_mixed", "f2": 7}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v4_mixed", "f2": 8}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v5_mixed", "f2": 9}'
+
+ - do:
+     search:
+        index: test_index
+
+ - match: { hits.total: 10 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/old_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/old_cluster/10_basic.yaml
new file mode 100644
index 00000000000..de2ca8206ec
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/old_cluster/10_basic.yaml
@@ -0,0 +1,22 @@
+---
+"Index data and search on the old cluster":
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v1_old", "f2": 0}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v2_old", "f2": 1}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v3_old", "f2": 2}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v4_old", "f2": 3}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v5_old", "f2": 4}'
+
+ - do:
+     search:
+        index: test_index
+
+ - match: { hits.total: 5 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/upgraded_cluster/10_basic.yaml
new file mode 100644
index 00000000000..edf9cb703d0
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/upgraded_cluster/10_basic.yaml
@@ -0,0 +1,22 @@
+---
+"Index data and search on the upgraded cluster":
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v1_upgraded", "f2": 10}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v2_upgraded", "f2": 11}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v3_upgraded", "f2": 12}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v4_upgraded", "f2": 13}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "v5_upgraded", "f2": 14}'
+
+ - do:
+     search:
+        index: test_index
+
+ - match: { hits.total: 15 }

From a844b085f18dbd12b3b7772758998171db8db8dd Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 6 Sep 2016 15:51:14 -0700
Subject: [PATCH 03/15] Made node config always have a unicast transport uri
 closure

---
 .../gradle/test/ClusterConfiguration.groovy   | 27 +++++++++++++------
 .../gradle/test/ClusterFormationTasks.groovy  | 23 +++-------------
 .../elasticsearch/gradle/test/NodeInfo.groovy | 16 ++++++++---
 .../gradle/test/RestSpecHack.groovy           |  8 ++++--
 qa/rolling-upgrade/build.gradle               | 25 ++++++++---------
 5 files changed, 53 insertions(+), 46 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index 689b7ed9ef4..1373eabfa45 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -45,9 +45,12 @@ class ClusterConfiguration {
     @Input
     int transportPort = 0
 
-    /** An override of the data directory. This may only be used with a single node. */
+    /**
+     * An override of the data directory. This may only be used with a single node.
+     * The value is lazily evaluated at runtime as a String path.
+     */
     @Input
-    File dataDir = null
+    Object dataDir = null
 
     /** Optional override of the cluster name. */
     @Input
@@ -65,16 +68,24 @@ class ClusterConfiguration {
             " " + System.getProperty('tests.jvm.argline', '')
 
     /**
-     * A uri that should be used for the unicast host list.
+     * A closure to call which returns the unicast host to connect to for cluster formation.
      *
      * This allows multi node clusters, or a new cluster to connect to an existing cluster.
-     * The type is Object to allow lazy evaluation. Typically this would be set with a
-     * closure in a GString like:
-     *
-     * {@code "$&#123;-> node.transportUri()&#125;"}
+     * The closure takes two arguments, the NodeInfo for the first node in the cluster, and
+     * an AntBuilder which may be used to wait on conditions before returning.
      */
     @Input
-    Closure unicastTransportUri = null
+    Closure unicastTransportUri = { NodeInfo seedNode, NodeInfo node, AntBuilder ant ->
+        if (seedNode == node) {
+            return null
+        }
+        ant.waitfor(maxwait: '20', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
+            resourceexists {
+                file(file: seedNode.transportPortsFile.toString())
+            }
+        }
+        return seedNode.transportUri()
+    }
 
     /**
      * A closure to call before the cluster is considered ready. The closure is passed the node info,
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 58b87e5c2d5..2e6ad2ece3f 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -86,7 +86,6 @@ class ClusterFormationTasks {
             configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchBwcDistro, config.bwcVersion)
         }
 
-        NodeInfo seedNode = null
         for (int i = 0; i < config.numNodes; ++i) {
             // we start N nodes and out of these N nodes there might be M bwc nodes.
             // for each of those nodes we might have a different configuratioon
@@ -97,10 +96,7 @@ class ClusterFormationTasks {
             }
             NodeInfo node = new NodeInfo(config, i, project, task, elasticsearchVersion, sharedDir)
             nodes.add(node)
-            if (i == 0) {
-                seedNode = node
-            }
-            startTasks.add(configureNode(project, task, cleanup, node, distro, seedNode))
+            startTasks.add(configureNode(project, task, cleanup, node, distro, nodes.get(0)))
         }
 
         Task wait = configureWaitTask("${task.name}#wait", project, nodes, startTasks)
@@ -263,20 +259,9 @@ class ClusterFormationTasks {
 
         Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
         writeConfig.doFirst {
-            if (node.config.unicastTransportUri != null) {
-                // if the unicast transport uri was specified, use it for all nodes
-                // this will typically be the case if all the nodes we are setting up
-                // should connect to a master in an already formed cluster
-                esConfig['discovery.zen.ping.unicast.hosts'] = node.config.unicastTransportUri()
-            } else if (node.nodeNum > 0) { // multi-node cluster case, we have to wait for the seed node to startup
-                ant.waitfor(maxwait: '20', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond') {
-                    resourceexists {
-                        file(file: seedNode.transportPortsFile.toString())
-                    }
-                }
-                // the seed node is enough to form the cluster - all subsequent nodes will get the seed node as a unicast
-                // host and join the cluster via that.
-                esConfig['discovery.zen.ping.unicast.hosts'] = "\"${seedNode.transportUri()}\""
+            String unicastTransportUri = node.config.unicastTransportUri(seedNode, node, project.ant)
+            if (unicastTransportUri != null) {
+                esConfig['discovery.zen.ping.unicast.hosts'] = unicastTransportUri
             }
             File configFile = new File(node.confDir, 'elasticsearch.yml')
             logger.info("Configuring ${configFile}")
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index e1834e26fb1..9a91c5ba138 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -57,8 +57,8 @@ class NodeInfo {
     /** config directory */
     File confDir
 
-    /** data directory */
-    File dataDir
+    /** data directory (as an Object, to allow lazy evaluation) */
+    Object dataDir
 
     /** THE config file */
     File configFile
@@ -155,7 +155,7 @@ class NodeInfo {
             }
         }
         env.put('ES_JVM_OPTIONS', new File(confDir, 'jvm.options'))
-        args.addAll("-E", "path.conf=${confDir}", "-E", "path.data=${dataDir}")
+        args.addAll("-E", "path.conf=${confDir}", "-E", "path.data=${-> dataDir.toString()}")
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {
             args.add('"') // end the entire command, quoted
         }
@@ -199,6 +199,16 @@ class NodeInfo {
         return transportPortsFile.readLines("UTF-8").get(0)
     }
 
+    /** Returns the file which contains the transport protocol ports for this node */
+    File getTransportPortsFile() {
+        return transportPortsFile
+    }
+
+    /** Returns the data directory for this node */
+    File getDataDir() {
+        return dataDir
+    }
+
     /** Returns the directory elasticsearch home is contained in for the given distribution */
     static File homeDir(File baseDir, String distro, String nodeVersion) {
         String path
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy
index 43b5c2f6f38..296ae711578 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestSpecHack.groovy
@@ -43,18 +43,22 @@ public class RestSpecHack {
     }
 
     /**
-     * Creates a task to copy the rest spec files.
+     * Creates a task (if necessary) to copy the rest spec files.
      *
      * @param project The project to add the copy task to
      * @param includePackagedTests true if the packaged tests should be copied, false otherwise
      */
     public static Task configureTask(Project project, boolean includePackagedTests) {
+        Task copyRestSpec = project.tasks.findByName('copyRestSpec')
+        if (copyRestSpec != null) {
+            return copyRestSpec
+        }
         Map copyRestSpecProps = [
                 name     : 'copyRestSpec',
                 type     : Copy,
                 dependsOn: [project.configurations.restSpec, 'processTestResources']
         ]
-        Task copyRestSpec = project.tasks.create(copyRestSpecProps) {
+        copyRestSpec = project.tasks.create(copyRestSpecProps) {
             from { project.zipTree(project.configurations.restSpec.singleFile) }
             include 'rest-api-spec/api/**'
             if (includePackagedTests) {
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 8fb53093585..e0ce18fb488 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -17,50 +17,47 @@
  * under the License.
  */
 
-
 import org.elasticsearch.gradle.test.NodeInfo
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
-RestIntegTestTask oldClusterTask = task oldClusterTest(type: RestIntegTestTask) {
+apply plugin: 'elasticsearch.standalone-test'
+
+task oldClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
   cluster {
     distribution = 'zip'
     bwcVersion = '2.4.0' // TODO: either randomize, or make this settable with sysprop
     numBwcNodes = 2
-    numNodes = 0
+    numNodes = 2
     clusterName = 'rolling-upgrade'
   }
   systemProperty 'tests.rest.suite', 'old_cluster'
 }
 
 task mixedClusterTest(type: RestIntegTestTask) {
-  dependsOn(oldClusterTest, 'oldClusterTest.node1') // TODO: what does this `oldClusterTest.node1` do?  is it a task?
-  NodeInfo aliveNode = oldClusterTask.getNodes().get(0)
-  NodeInfo stoppedNode = oldClusterTask.getNodes().get(1)
+  dependsOn(oldClusterTest, 'oldClusterTest#node1.stop')
   cluster {
     distribution = 'zip'
     clusterName = 'rolling-upgrade'
-    unicastTransportUri = { -> aliveNode.transportUri() }
-    dataDir = stoppedNode.dataDir
+    unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
+    dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
 
 }
 
 task upgradedClusterTest(type: RestIntegTestTask) {
-  // stop alive node from oldClusterTest and get its dataDir, and get alive node for unicast host
-  NodeInfo stoppedNode = null
-  NodeInfo aliveNode = null
+  dependsOn(mixedClusterTest, 'oldClusterTest#node0.stop')
   cluster {
     distribution = 'zip'
     clusterName = 'rolling-upgrade'
-    unicastTransportUri = { -> aliveNode.transportUri() }
-    dataDir = stoppedNode.dataDir
+    unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
+    dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
   }
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
 }
 
 task integTest {
-  dependsOn = [upgradedClusterTest]
+  dependsOn = [oldClusterTest, mixedClusterTest, upgradedClusterTest]
 }
 check.dependsOn(integTest)

From bd2de367ccb189c02c0a77d85f549d8cde8bdeee Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 6 Sep 2016 16:08:27 -0700
Subject: [PATCH 04/15] Use 5.0 alpha5 for bwc version, so we have
 transport.ports

Also fixed bug to ensure unicast host is writtent in yaml quotes
---
 .../org/elasticsearch/gradle/test/ClusterFormationTasks.groovy  | 2 +-
 qa/rolling-upgrade/build.gradle                                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 2e6ad2ece3f..208edd63d1d 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -261,7 +261,7 @@ class ClusterFormationTasks {
         writeConfig.doFirst {
             String unicastTransportUri = node.config.unicastTransportUri(seedNode, node, project.ant)
             if (unicastTransportUri != null) {
-                esConfig['discovery.zen.ping.unicast.hosts'] = unicastTransportUri
+                esConfig['discovery.zen.ping.unicast.hosts'] = "\"${unicastTransportUri}\""
             }
             File configFile = new File(node.confDir, 'elasticsearch.yml')
             logger.info("Configuring ${configFile}")
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index e0ce18fb488..7280ddd1271 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -26,7 +26,7 @@ task oldClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
   cluster {
     distribution = 'zip'
-    bwcVersion = '2.4.0' // TODO: either randomize, or make this settable with sysprop
+    bwcVersion = '5.0.0-alpha5' // TODO: either randomize, or make this settable with sysprop
     numBwcNodes = 2
     numNodes = 2
     clusterName = 'rolling-upgrade'

From 0b3eb11712f97a7f50a0a12c8d88065d4007a46a Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Mon, 12 Sep 2016 16:05:36 -0400
Subject: [PATCH 05/15] Changed rest-api-spec tests folder structure

---
 qa/rolling-upgrade/build.gradle                               | 1 -
 .../test}/mixed_cluster/10_basic.yaml                         | 0
 .../test}/old_cluster/10_basic.yaml                           | 4 ++--
 .../test}/upgraded_cluster/10_basic.yaml                      | 0
 4 files changed, 2 insertions(+), 3 deletions(-)
 rename qa/rolling-upgrade/src/test/resources/{rest-api-spec.test => rest-api-spec/test}/mixed_cluster/10_basic.yaml (100%)
 rename qa/rolling-upgrade/src/test/resources/{rest-api-spec.test => rest-api-spec/test}/old_cluster/10_basic.yaml (97%)
 rename qa/rolling-upgrade/src/test/resources/{rest-api-spec.test => rest-api-spec/test}/upgraded_cluster/10_basic.yaml (100%)

diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 7280ddd1271..acc59bb3bed 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -17,7 +17,6 @@
  * under the License.
  */
 
-import org.elasticsearch.gradle.test.NodeInfo
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
 apply plugin: 'elasticsearch.standalone-test'
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
similarity index 100%
rename from qa/rolling-upgrade/src/test/resources/rest-api-spec.test/mixed_cluster/10_basic.yaml
rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/old_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
similarity index 97%
rename from qa/rolling-upgrade/src/test/resources/rest-api-spec.test/old_cluster/10_basic.yaml
rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
index de2ca8206ec..76c6adda21e 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/old_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
@@ -1,7 +1,7 @@
 ---
 "Index data and search on the old cluster":
-  - do:
-      bulk:
+ - do:
+     bulk:
         refresh: true
         body:
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec.test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
similarity index 100%
rename from qa/rolling-upgrade/src/test/resources/rest-api-spec.test/upgraded_cluster/10_basic.yaml
rename to qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml

From 0c7dd1865c4772731b826290b2f32e9d26603592 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Mon, 12 Sep 2016 18:46:06 -0400
Subject: [PATCH 06/15] Reworking yaml tests for rolling upgrades

---
 .../org/elasticsearch/gradle/test/NodeInfo.groovy    |  3 +++
 qa/rolling-upgrade/build.gradle                      |  4 ++--
 .../rest-api-spec/test/mixed_cluster/10_basic.yaml   | 12 +++++++++---
 .../test/upgraded_cluster/10_basic.yaml              | 12 +++++++++---
 4 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index 9a91c5ba138..85af2debf1b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -206,6 +206,9 @@ class NodeInfo {
 
     /** Returns the data directory for this node */
     File getDataDir() {
+        if (!(dataDir instanceof File)) {
+            return new File(dataDir)
+        }
         return dataDir
     }
 
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index acc59bb3bed..456db46d666 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -39,7 +39,7 @@ task mixedClusterTest(type: RestIntegTestTask) {
     distribution = 'zip'
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
-    dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
+    dataDir = "${-> oldClusterTest.nodes[1].getDataDir()}"
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
 
@@ -51,7 +51,7 @@ task upgradedClusterTest(type: RestIntegTestTask) {
     distribution = 'zip'
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
-    dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
+    dataDir = "${-> oldClusterTest.nodes[0].getDataDir()}"
   }
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
index a46d7aaf044..dcfa946ed64 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
@@ -1,7 +1,13 @@
 ---
 "Index data and search on the mixed cluster":
-  - do:
-      bulk:
+ - do:
+     search:
+        index: test_index
+
+ - match: { hits.total: 5 } # no new indexed data, so expect the original 5 documents from the old cluster
+
+ - do:
+     bulk:
         refresh: true
         body:
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
@@ -19,4 +25,4 @@
      search:
         index: test_index
 
- - match: { hits.total: 10 }
+ - match: { hits.total: 10 } # 5 docs from old cluster, 5 docs from mixed cluster
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
index edf9cb703d0..f6606c758a0 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
@@ -1,7 +1,13 @@
 ---
 "Index data and search on the upgraded cluster":
-  - do:
-      bulk:
+ - do:
+     search:
+       index: test_index
+
+ - match: { hits.total: 10 } # no new indexed data, so expect the original 10 documents from the old and mixed clusters
+
+ - do:
+     bulk:
         refresh: true
         body:
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
@@ -19,4 +25,4 @@
      search:
         index: test_index
 
- - match: { hits.total: 15 }
+ - match: { hits.total: 15 } # 10 docs from previous clusters plus 5 new docs

From 513ed58d177466052efd278d3fed401d30cc3afa Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Tue, 13 Sep 2016 12:36:20 -0400
Subject: [PATCH 07/15] Added a indices flush to the rolling upgrades rest
 tests to ensure they are on disk for the upgraded nodes to pick up.

---
 qa/rolling-upgrade/build.gradle                               | 4 ++--
 .../resources/rest-api-spec/test/mixed_cluster/10_basic.yaml  | 4 ++++
 .../resources/rest-api-spec/test/old_cluster/10_basic.yaml    | 4 ++++
 .../rest-api-spec/test/upgraded_cluster/10_basic.yaml         | 4 ++++
 4 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 456db46d666..acc59bb3bed 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -39,7 +39,7 @@ task mixedClusterTest(type: RestIntegTestTask) {
     distribution = 'zip'
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
-    dataDir = "${-> oldClusterTest.nodes[1].getDataDir()}"
+    dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
 
@@ -51,7 +51,7 @@ task upgradedClusterTest(type: RestIntegTestTask) {
     distribution = 'zip'
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
-    dataDir = "${-> oldClusterTest.nodes[0].getDataDir()}"
+    dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
   }
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
index dcfa946ed64..7caca10911f 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
@@ -21,6 +21,10 @@
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
           - '{"f1": "v5_mixed", "f2": 9}'
 
+ - do:
+     indices.flush:
+        index: test_index
+
  - do:
      search:
         index: test_index
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
index 76c6adda21e..7230c67e3d2 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
@@ -15,6 +15,10 @@
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
           - '{"f1": "v5_old", "f2": 4}'
 
+ - do:
+     indices.flush:
+        index: test_index
+
  - do:
      search:
         index: test_index
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
index f6606c758a0..a0a68b00987 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
@@ -21,6 +21,10 @@
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
           - '{"f1": "v5_upgraded", "f2": 14}'
 
+ - do:
+     indices.flush:
+        index: test_index
+
  - do:
      search:
         index: test_index

From 3f7987404243ee1ae63b80436fb195b7ad868c36 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Wed, 14 Sep 2016 23:34:19 -0400
Subject: [PATCH 08/15] Prevent the rolling upgrades rest tests from cleaning
 up indices after finishing if a the tests.rest.preserve_indices system
 property is set

---
 qa/rolling-upgrade/build.gradle                 |  3 ++-
 .../test/mixed_cluster/10_basic.yaml            | 10 ++++++++++
 .../test/old_cluster/10_basic.yaml              |  8 ++++++++
 .../test/upgraded_cluster/10_basic.yaml         |  4 ++++
 .../elasticsearch/test/rest/ESRestTestCase.java | 17 ++++++++++-------
 5 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index acc59bb3bed..a5c5dccaf62 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -31,6 +31,7 @@ task oldClusterTest(type: RestIntegTestTask) {
     clusterName = 'rolling-upgrade'
   }
   systemProperty 'tests.rest.suite', 'old_cluster'
+  systemProperty 'tests.rest.preserve_indices', 'true'
 }
 
 task mixedClusterTest(type: RestIntegTestTask) {
@@ -42,7 +43,7 @@ task mixedClusterTest(type: RestIntegTestTask) {
     dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
-
+  systemProperty 'tests.rest.preserve_indices', 'true'
 }
 
 task upgradedClusterTest(type: RestIntegTestTask) {
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
index 7caca10911f..495e1d0b3d7 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
@@ -1,5 +1,15 @@
 ---
 "Index data and search on the mixed cluster":
+ - do:
+     cluster.health:
+        wait_for_status: green
+
+ - do:
+      indices.get_settings:
+        index: test_index
+
+ - match: { test_index.settings.index.number_of_replicas: "0" }
+
  - do:
      search:
         index: test_index
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
index 7230c67e3d2..f1f90cf9d22 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
@@ -1,5 +1,13 @@
 ---
 "Index data and search on the old cluster":
+ - do:
+     indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+
  - do:
      bulk:
         refresh: true
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
index a0a68b00987..b7a38b4123b 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
@@ -1,5 +1,9 @@
 ---
 "Index data and search on the upgraded cluster":
+ - do:
+     cluster.health:
+        wait_for_status: green
+
  - do:
      search:
        index: test_index
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 573c301105a..be67017c48d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -138,13 +138,16 @@ public class ESRestTestCase extends ESTestCase {
     }
 
     private void wipeCluster() throws IOException {
-        // wipe indices
-        try {
-            adminClient().performRequest("DELETE", "*");
-        } catch (ResponseException e) {
-            // 404 here just means we had no indexes
-            if (e.getResponse().getStatusLine().getStatusCode() != 404) {
-                throw e;
+        final boolean preserveIndices = Boolean.parseBoolean(System.getProperty("tests.rest.preserve_indices"));
+        if (preserveIndices == false) {
+            // wipe indices
+            try {
+                adminClient().performRequest("DELETE", "*");
+            } catch (ResponseException e) {
+                // 404 here just means we had no indexes
+                if (e.getResponse().getStatusLine().getStatusCode() != 404) {
+                    throw e;
+                }
             }
         }
 

From ec7e383783ce47d1c96e02e54c5918e55892115d Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Thu, 15 Sep 2016 15:08:11 -0400
Subject: [PATCH 09/15] Better checks for the cluster being up in the rolling
 upgrades tests.

---
 .../rest-api-spec/test/mixed_cluster/10_basic.yaml          | 6 +-----
 .../rest-api-spec/test/upgraded_cluster/10_basic.yaml       | 2 ++
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
index 495e1d0b3d7..62fe1c77627 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
@@ -4,11 +4,7 @@
      cluster.health:
         wait_for_status: green
 
- - do:
-      indices.get_settings:
-        index: test_index
-
- - match: { test_index.settings.index.number_of_replicas: "0" }
+ - match: { number_of_nodes: 2 }
 
  - do:
      search:
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
index b7a38b4123b..eec2820ce32 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
@@ -4,6 +4,8 @@
      cluster.health:
         wait_for_status: green
 
+ - match: { number_of_nodes: 2 }
+
  - do:
      search:
        index: test_index

From ba072ec18e59dc1c5461c0211327439ddda11eaf Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Thu, 15 Sep 2016 18:52:15 -0400
Subject: [PATCH 10/15] When Elasticsearch nodes are started in gradle to form
 a cluster, we wait for the cluster health to indicate the necessary nodes
 have formed a cluster.  This check was an exact value (equality) check. 
 However, if we are trying to connect the nodes in the cluster to nodes from a
 previously formed cluster (of the same name), then we will have more nodes
 returned by the cluster health check than the current task's configured
 number of nodes.  Hence, this check needs to be a >= check.  This commit
 fixes it.

---
 .../elasticsearch/gradle/test/ClusterConfiguration.groovy | 1 +
 qa/rolling-upgrade/build.gradle                           | 8 ++++++--
 .../rest-api-spec/test/mixed_cluster/10_basic.yaml        | 3 +--
 .../rest-api-spec/test/upgraded_cluster/10_basic.yaml     | 3 +--
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index 1373eabfa45..94893126a7c 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -95,6 +95,7 @@ class ClusterConfiguration {
     @Input
     Closure waitCondition = { NodeInfo node, AntBuilder ant ->
         File tmpFile = new File(node.cwd, 'wait.success')
+        ant.echo("==> Current time: " + new Date());
         ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}",
                 dest: tmpFile.toString(),
                 ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index a5c5dccaf62..a5cb25c0038 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -25,8 +25,12 @@ task oldClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
   cluster {
     distribution = 'zip'
-    bwcVersion = '5.0.0-alpha5' // TODO: either randomize, or make this settable with sysprop
-    numBwcNodes = 2
+    // TODO: Right now, this just forms a cluster with the current version of ES,
+    // because we don't support clusters with nodes on different alpha/beta releases of ES.
+    // When the GA is released, we should change the bwcVersion to 5.0.0 and uncomment
+    // numBwcNodes = 2
+    //bwcVersion = '5.0.0-alpha5' // TODO: either randomize, or make this settable with sysprop
+    //numBwcNodes = 2
     numNodes = 2
     clusterName = 'rolling-upgrade'
   }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
index 62fe1c77627..a2b40cc54f7 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
@@ -3,8 +3,7 @@
  - do:
      cluster.health:
         wait_for_status: green
-
- - match: { number_of_nodes: 2 }
+        wait_for_nodes: 2
 
  - do:
      search:
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
index eec2820ce32..03dcdc583d3 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
@@ -3,8 +3,7 @@
  - do:
      cluster.health:
         wait_for_status: green
-
- - match: { number_of_nodes: 2 }
+        wait_for_nodes: 2
 
  - do:
      search:

From 56f97500c61cc9e5facc7a6b75a0f4cef57f9092 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 16 Sep 2016 10:14:51 -0400
Subject: [PATCH 11/15] In the rolling upgrades tests, we do not want to stop
 nodes automatically between tasks, as we want some of the nodes from the
 previous task to continue running in the next task. This commit enables a
 cluster configuration setting to not stop nodes automatically after a task
 runs, but instead the creator of the test task must stop the running nodes
 explicitly in a cleanup phase.

---
 .../gradle/test/ClusterConfiguration.groovy       | 15 +++++++++++++--
 .../gradle/test/ClusterFormationTasks.groovy      |  4 ++--
 qa/rolling-upgrade/build.gradle                   |  7 ++++++-
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index 94893126a7c..b05929ac4c3 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -62,6 +62,14 @@ class ClusterConfiguration {
     @Input
     boolean debug = false
 
+    /**
+     * Whether to stop the nodes in the cluster upon task completion. The only reason to
+     * set this to false is if you want the nodes in the cluster to hang around and you
+     * will clean them up later by calling the `taskName#nodeX.stop` task explicitly.
+     */
+    @Input
+    boolean stopNodesOnCompletion = true
+
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
             " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
@@ -95,8 +103,11 @@ class ClusterConfiguration {
     @Input
     Closure waitCondition = { NodeInfo node, AntBuilder ant ->
         File tmpFile = new File(node.cwd, 'wait.success')
-        ant.echo("==> Current time: " + new Date());
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}",
+        ant.echo("==> [${new Date()}] checking health: http://${node.httpUri()}/_cluster/health?wait_for_nodes>=${numNodes}")
+        // checking here for wait_for_nodes to be >= the number of nodes because its possible
+        // this cluster is attempting to connect to nodes created by another task (same cluster name),
+        // so there will be more nodes in that case in the cluster state
+        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes>=${numNodes}",
                 dest: tmpFile.toString(),
                 ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task
                 retries: 10)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 208edd63d1d..d89f094acdc 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -173,9 +173,9 @@ class ClusterFormationTasks {
 
         Task start = configureStartTask(taskName(task, node, 'start'), project, setup, node)
 
-        if (node.config.daemonize) {
+        Task stop = configureStopTask(taskName(task, node, 'stop'), project, [], node)
+        if (node.config.daemonize && node.config.stopNodesOnCompletion) {
             // if we are running in the background, make sure to stop the server when the task completes
-            Task stop = configureStopTask(taskName(task, node, 'stop'), project, [], node)
             task.finalizedBy(stop)
         }
         return start
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index a5cb25c0038..227d526a2ef 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -33,6 +33,7 @@ task oldClusterTest(type: RestIntegTestTask) {
     //numBwcNodes = 2
     numNodes = 2
     clusterName = 'rolling-upgrade'
+    stopNodesOnCompletion = false
   }
   systemProperty 'tests.rest.suite', 'old_cluster'
   systemProperty 'tests.rest.preserve_indices', 'true'
@@ -45,6 +46,7 @@ task mixedClusterTest(type: RestIntegTestTask) {
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
     dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
+    stopNodesOnCompletion = false
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
   systemProperty 'tests.rest.preserve_indices', 'true'
@@ -61,7 +63,10 @@ task upgradedClusterTest(type: RestIntegTestTask) {
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
 }
 
+// only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
+upgradedClusterTest.finalizedBy 'mixedClusterTest#stop'
+
 task integTest {
-  dependsOn = [oldClusterTest, mixedClusterTest, upgradedClusterTest]
+  dependsOn = [upgradedClusterTest]
 }
 check.dependsOn(integTest)

From 22e6bc835944df7bd7efdd8431ea459f74662ac3 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 16 Sep 2016 11:32:28 -0400
Subject: [PATCH 12/15] Disables unit tests for rolling upgrades, as there are
 only rest integration tests.

---
 qa/rolling-upgrade/build.gradle | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 227d526a2ef..809c2376e01 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -69,4 +69,7 @@ upgradedClusterTest.finalizedBy 'mixedClusterTest#stop'
 task integTest {
   dependsOn = [upgradedClusterTest]
 }
+
+test.enabled = false // no unit tests for rolling upgrades, only the rest integration test
+
 check.dependsOn(integTest)

From 83adc8701519406ccc67ad45bf0b447706c95012 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 16 Sep 2016 16:32:43 -0400
Subject: [PATCH 13/15] Removes stopNodeUponCompletion in favor of moving the
 stop nodes task to the final part of the cluster test task execution graph.

---
 .../elasticsearch/gradle/test/ClusterConfiguration.groovy | 8 --------
 .../gradle/test/ClusterFormationTasks.groovy              | 5 +++--
 qa/rolling-upgrade/build.gradle                           | 8 +++-----
 3 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index b05929ac4c3..79b3d555070 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -62,14 +62,6 @@ class ClusterConfiguration {
     @Input
     boolean debug = false
 
-    /**
-     * Whether to stop the nodes in the cluster upon task completion. The only reason to
-     * set this to false is if you want the nodes in the cluster to hang around and you
-     * will clean them up later by calling the `taskName#nodeX.stop` task explicitly.
-     */
-    @Input
-    boolean stopNodesOnCompletion = true
-
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
             " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index d89f094acdc..6d3c7412dac 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -173,10 +173,11 @@ class ClusterFormationTasks {
 
         Task start = configureStartTask(taskName(task, node, 'start'), project, setup, node)
 
-        Task stop = configureStopTask(taskName(task, node, 'stop'), project, [], node)
-        if (node.config.daemonize && node.config.stopNodesOnCompletion) {
+        if (node.config.daemonize) {
+            Task stop = configureStopTask(taskName(task, node, 'stop'), project, [], node)
             // if we are running in the background, make sure to stop the server when the task completes
             task.finalizedBy(stop)
+            start.finalizedBy(stop)
         }
         return start
     }
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 809c2376e01..bb6697dbdfb 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -33,7 +33,6 @@ task oldClusterTest(type: RestIntegTestTask) {
     //numBwcNodes = 2
     numNodes = 2
     clusterName = 'rolling-upgrade'
-    stopNodesOnCompletion = false
   }
   systemProperty 'tests.rest.suite', 'old_cluster'
   systemProperty 'tests.rest.preserve_indices', 'true'
@@ -46,10 +45,10 @@ task mixedClusterTest(type: RestIntegTestTask) {
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
     dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
-    stopNodesOnCompletion = false
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
   systemProperty 'tests.rest.preserve_indices', 'true'
+  finalizedBy 'oldClusterTest#node0.stop'
 }
 
 task upgradedClusterTest(type: RestIntegTestTask) {
@@ -61,11 +60,10 @@ task upgradedClusterTest(type: RestIntegTestTask) {
     dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
   }
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
+  // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
+  finalizedBy 'mixedClusterTest#stop'
 }
 
-// only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
-upgradedClusterTest.finalizedBy 'mixedClusterTest#stop'
-
 task integTest {
   dependsOn = [upgradedClusterTest]
 }

From 98230d035a8dd1e8ae0a82058db73a9f8d03d66e Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 16 Sep 2016 19:19:42 -0400
Subject: [PATCH 14/15] Adds a preserveIndicesUponCompletion method to
 ESRestTestCase that can be overridden by subclasses if the test must not
 delete indices it created after exiting.

---
 .../gradle/test/ClusterFormationTasks.groovy      |  3 +--
 qa/rolling-upgrade/build.gradle                   |  2 --
 .../UpgradeClusterClientYamlTestSuiteIT.java      |  5 +++++
 .../elasticsearch/test/rest/ESRestTestCase.java   | 15 +++++++++++++--
 4 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 6d3c7412dac..cd7fc8fa877 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -46,7 +46,7 @@ class ClusterFormationTasks {
     /**
      * Adds dependent tasks to the given task to start and stop a cluster with the given configuration.
      *
-     * Returns a NodeInfo object for the first node in the cluster.
+     * Returns a list of NodeInfo objects for each node in the cluster.
      */
     static List<NodeInfo> setup(Project project, Task task, ClusterConfiguration config) {
         if (task.getEnabled() == false) {
@@ -102,7 +102,6 @@ class ClusterFormationTasks {
         Task wait = configureWaitTask("${task.name}#wait", project, nodes, startTasks)
         task.dependsOn(wait)
 
-        // delay the resolution of the uri by wrapping in a closure, so it is not used until read for tests
         return nodes
     }
 
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index bb6697dbdfb..f90763a12dd 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -35,7 +35,6 @@ task oldClusterTest(type: RestIntegTestTask) {
     clusterName = 'rolling-upgrade'
   }
   systemProperty 'tests.rest.suite', 'old_cluster'
-  systemProperty 'tests.rest.preserve_indices', 'true'
 }
 
 task mixedClusterTest(type: RestIntegTestTask) {
@@ -47,7 +46,6 @@ task mixedClusterTest(type: RestIntegTestTask) {
     dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
-  systemProperty 'tests.rest.preserve_indices', 'true'
   finalizedBy 'oldClusterTest#node0.stop'
 }
 
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
index f5cc3367202..bfb3c63c3da 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
@@ -31,6 +31,11 @@ import java.io.IOException;
 @TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
 public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
     public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index e135fc27ea6..1e419faf06b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -114,6 +114,7 @@ public class ESRestTestCase extends ESTestCase {
         }
     }
 
+
     /**
      * Clean up after the test case.
      */
@@ -138,9 +139,19 @@ public class ESRestTestCase extends ESTestCase {
         return adminClient;
     }
 
+    /**
+     * Returns whether to preserve the indices created during this test on completion of this test.
+     * Defaults to {@code false}. Override this method if indices should be preserved after the test,
+     * with the assumption that some other process or test will clean up the indices afterward.
+     * This is useful if the data directory and indices need to be preserved between test runs
+     * (for example, when testing rolling upgrades).
+     */
+    protected boolean preserveIndicesUponCompletion() {
+        return false;
+    }
+
     private void wipeCluster() throws IOException {
-        final boolean preserveIndices = Boolean.parseBoolean(System.getProperty("tests.rest.preserve_indices"));
-        if (preserveIndices == false) {
+        if (preserveIndicesUponCompletion() == false) {
             // wipe indices
             try {
                 adminClient().performRequest("DELETE", "*");

From 8afc83047fbf8562664cfba8a5d7ab3056b819cc Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Mon, 19 Sep 2016 15:45:41 -0400
Subject: [PATCH 15/15] Change the timeout of the rolling upgrades test from 40
 mins to 5 mins to still allow accounting for slow VMs

---
 .../upgrades/UpgradeClusterClientYamlTestSuiteIT.java           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
index bfb3c63c3da..496a02e42dc 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
@@ -28,7 +28,7 @@ import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
-@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
+@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
 public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     @Override