diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 58e7e50e421..905710a82be 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -40,8 +40,9 @@ List<String> availableBoxes = [
     'ubuntu-1504'
 ]
 
+String vagrantBoxes = getProperties().get('vagrant.boxes', 'sample')
 List<String> boxes = []
-for (String box : getProperties().get('vagrant.boxes', 'sample').split(',')) {
+for (String box : vagrantBoxes.split(',')) {
   if (box == 'sample') {
     boxes.add('centos-7')
     boxes.add('ubuntu-1404')
@@ -56,9 +57,28 @@ for (String box : getProperties().get('vagrant.boxes', 'sample').split(',')) {
   }
 }
 
-/* The version of elasticsearch that we upgrade *from* as part of testing
- * upgrades. */
-String upgradeFromVersion = '2.3.3'
+long seed
+String formattedSeed
+String[] upgradeFromVersions
+String upgradeFromVersion
+
+String maybeTestsSeed = System.getProperty("tests.seed", null);
+if (maybeTestsSeed != null) {
+  seed = new BigInteger(maybeTestsSeed, 16).longValue()
+  formattedSeed = maybeTestsSeed
+} else {
+  seed = new Random().nextLong()
+  formattedSeed = String.format("%016X", seed)
+}
+
+String maybeUpdradeFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
+if (maybeUpdradeFromVersions != null) {
+  upgradeFromVersions = maybeUpdradeFromVersions.split(",")
+} else {
+  upgradeFromVersions = new File(project.projectDir, 'versions')
+}
+
+upgradeFromVersion = upgradeFromVersions[new Random(seed).nextInt(upgradeFromVersions.length)]
 
 configurations {
   test
@@ -95,8 +115,37 @@ task stop {
   description 'Stop any tasks from tests that still may be running'
 }
 
+Set<String> getVersions() {
+  Node xml
+  new URL('http://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
+    xml = new XmlParser().parse(s)
+  }
+  return new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /2\.\d\.\d/ })
+}
+
+task updatePackagingTestUpgradeFromVersions {
+  doLast {
+    Set<String> versions = getVersions()
+    new File(project.projectDir, 'versions').text = versions.join('\n') + '\n'
+  }
+}
+
+task verifyPackagingTestUpgradeFromVersions {
+  doLast {
+    String maybeUpdateFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
+    if (maybeUpdateFromVersions == null) {
+      Set<String> versions = getVersions()
+      Set<String> actualVersions = new HashSet<>(Arrays.asList(upgradeFromVersions))
+      if (!versions.equals(actualVersions)) {
+        throw new GradleException("out-of-date versions [" + actualVersions + "], expected [" + versions + "]; run gradle updatePackagingTestUpgradeFromVersions")
+      }
+    }
+  }
+}
+
 File testRoot = new File("$buildDir/testroot")
 task createTestRoot {
+  dependsOn verifyPackagingTestUpgradeFromVersions
   outputs.dir testRoot
   doLast {
     testRoot.mkdirs()
@@ -147,6 +196,14 @@ task packagingTest {
               "    'sample' can be used to test a single yum and apt box. 'all' can be used to\n" +
               "    test all available boxes. The available boxes are: \n" +
               "    ${availableBoxes}"
+  gradle.addBuildListener new BuildAdapter() {
+    @Override
+    void buildFinished(BuildResult result) {
+      if (result.failure) {
+        println "Reproduce with: gradle packagingTest -Pvagrant.boxes=${vagrantBoxes} -Dtests.seed=${formattedSeed} -Dtests.packaging.upgrade.from.versions=${upgradeFromVersions.join(",")}"
+      }
+    }
+  }
 }
 
 // Each box gets it own set of tasks
@@ -199,4 +256,3 @@ for (String box : availableBoxes) {
   }
   packagingTest.dependsOn(packaging)
 }
-
diff --git a/qa/vagrant/versions b/qa/vagrant/versions
new file mode 100644
index 00000000000..5f6e0edf99c
--- /dev/null
+++ b/qa/vagrant/versions
@@ -0,0 +1,13 @@
+2.0.0
+2.0.1
+2.0.2
+2.1.0
+2.1.1
+2.1.2
+2.2.0
+2.2.1
+2.2.2
+2.3.0
+2.3.1
+2.3.2
+2.3.3