From 3f5b9943fd39c99d9fcfa130e92cb3883253a527 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 27 Jun 2016 12:13:06 -0400
Subject: [PATCH] Randomize packaging upgrade test

This commit adds randomization for the packaging upgrade test. In
particular, we extract a list of the released version of Elasticsearch
from Maven Central and randomize the selection of the version to upgrade
from. The randomization is repeatable, and supports the tests.seed
property. Specific versions can be tested by setting the property
tests.packaging.upgrade.from.versions.

Relates #19033
---
 qa/vagrant/build.gradle | 66 +++++++++++++++++++++++++++++++++++++----
 qa/vagrant/versions     | 13 ++++++++
 2 files changed, 74 insertions(+), 5 deletions(-)
 create mode 100644 qa/vagrant/versions

diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 58e7e50e421..905710a82be 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -40,8 +40,9 @@ List<String> availableBoxes = [
     'ubuntu-1504'
 ]
 
+String vagrantBoxes = getProperties().get('vagrant.boxes', 'sample')
 List<String> boxes = []
-for (String box : getProperties().get('vagrant.boxes', 'sample').split(',')) {
+for (String box : vagrantBoxes.split(',')) {
   if (box == 'sample') {
     boxes.add('centos-7')
     boxes.add('ubuntu-1404')
@@ -56,9 +57,28 @@ for (String box : getProperties().get('vagrant.boxes', 'sample').split(',')) {
   }
 }
 
-/* The version of elasticsearch that we upgrade *from* as part of testing
- * upgrades. */
-String upgradeFromVersion = '2.3.3'
+long seed
+String formattedSeed
+String[] upgradeFromVersions
+String upgradeFromVersion
+
+String maybeTestsSeed = System.getProperty("tests.seed", null);
+if (maybeTestsSeed != null) {
+  seed = new BigInteger(maybeTestsSeed, 16).longValue()
+  formattedSeed = maybeTestsSeed
+} else {
+  seed = new Random().nextLong()
+  formattedSeed = String.format("%016X", seed)
+}
+
+String maybeUpdradeFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
+if (maybeUpdradeFromVersions != null) {
+  upgradeFromVersions = maybeUpdradeFromVersions.split(",")
+} else {
+  upgradeFromVersions = new File(project.projectDir, 'versions')
+}
+
+upgradeFromVersion = upgradeFromVersions[new Random(seed).nextInt(upgradeFromVersions.length)]
 
 configurations {
   test
@@ -95,8 +115,37 @@ task stop {
   description 'Stop any tasks from tests that still may be running'
 }
 
+Set<String> getVersions() {
+  Node xml
+  new URL('http://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
+    xml = new XmlParser().parse(s)
+  }
+  return new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /2\.\d\.\d/ })
+}
+
+task updatePackagingTestUpgradeFromVersions {
+  doLast {
+    Set<String> versions = getVersions()
+    new File(project.projectDir, 'versions').text = versions.join('\n') + '\n'
+  }
+}
+
+task verifyPackagingTestUpgradeFromVersions {
+  doLast {
+    String maybeUpdateFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
+    if (maybeUpdateFromVersions == null) {
+      Set<String> versions = getVersions()
+      Set<String> actualVersions = new HashSet<>(Arrays.asList(upgradeFromVersions))
+      if (!versions.equals(actualVersions)) {
+        throw new GradleException("out-of-date versions [" + actualVersions + "], expected [" + versions + "]; run gradle updatePackagingTestUpgradeFromVersions")
+      }
+    }
+  }
+}
+
 File testRoot = new File("$buildDir/testroot")
 task createTestRoot {
+  dependsOn verifyPackagingTestUpgradeFromVersions
   outputs.dir testRoot
   doLast {
     testRoot.mkdirs()
@@ -147,6 +196,14 @@ task packagingTest {
               "    'sample' can be used to test a single yum and apt box. 'all' can be used to\n" +
               "    test all available boxes. The available boxes are: \n" +
               "    ${availableBoxes}"
+  gradle.addBuildListener new BuildAdapter() {
+    @Override
+    void buildFinished(BuildResult result) {
+      if (result.failure) {
+        println "Reproduce with: gradle packagingTest -Pvagrant.boxes=${vagrantBoxes} -Dtests.seed=${formattedSeed} -Dtests.packaging.upgrade.from.versions=${upgradeFromVersions.join(",")}"
+      }
+    }
+  }
 }
 
 // Each box gets it own set of tasks
@@ -199,4 +256,3 @@ for (String box : availableBoxes) {
   }
   packagingTest.dependsOn(packaging)
 }
-
diff --git a/qa/vagrant/versions b/qa/vagrant/versions
new file mode 100644
index 00000000000..5f6e0edf99c
--- /dev/null
+++ b/qa/vagrant/versions
@@ -0,0 +1,13 @@
+2.0.0
+2.0.1
+2.0.2
+2.1.0
+2.1.1
+2.1.2
+2.2.0
+2.2.1
+2.2.2
+2.3.0
+2.3.1
+2.3.2
+2.3.3