diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index edad98b5fbd..78730f7a9f4 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -90,7 +90,18 @@ configurations {
 }
 
 repositories {
-  mavenCentral()
+  mavenCentral() // Try maven central first, it'll have releases before 5.0.0
+  /* Setup a repository that tries to download from
+    https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-[revision].[ext]
+    which should work for 5.0.0+. This isn't a real ivy repository so we have
+    to fake the ivy.xml file locally but it gets the job done. */
+  ivy {
+    url "https://artifacts.elastic.co/downloads/"
+    layout "pattern", {
+      artifact "elasticsearch/elasticsearch-[revision].[ext]"
+      ivy "file:$projectDir/ivy.xml"
+    }
+  }
 }
 
 dependencies {
@@ -150,8 +161,8 @@ task verifyPackagingTestUpgradeFromVersions {
       Set<String> versions = getVersions()
       Set<String> actualVersions = new TreeSet<>(Arrays.asList(upgradeFromVersions))
       if (!versions.equals(actualVersions)) {
-        println("out-of-date versions [" + actualVersions + "], expected [" + versions + "]; run gradle updatePackagingTestUpgradeFromVersions")
-        // throw new GradleException("out-of-date versions [" + actualVersions + "], expected [" + versions + "]; run gradle updatePackagingTestUpgradeFromVersions")
+        throw new GradleException("out-of-date versions " + actualVersions +
+          ", expected " + versions + "; run gradle updatePackagingTestUpgradeFromVersions")
       }
     }
   }
@@ -245,7 +256,7 @@ for (String box : availableBoxes) {
 
   Task up = tasks.create("vagrant${boxTask}#up", VagrantCommandTask) {
     boxName box
-    /* Its important that we try to reprovision the box even if it already
+    /* It's important that we try to reprovision the box even if it already
       exists. That way updates to the vagrant configuration take automatically.
       That isn't to say that the updates will always be compatible. Its ok to
       just destroy the boxes if they get busted but that is a manual step
diff --git a/qa/vagrant/ivy.xml b/qa/vagrant/ivy.xml
new file mode 100644
index 00000000000..63afcc52bcf
--- /dev/null
+++ b/qa/vagrant/ivy.xml
@@ -0,0 +1,6 @@
+<ivy-module version="2.0">
+  <!-- Cheating ivy repository so we can resolve deb and rpm artifacts from
+    https://artifacts.elastic.co/downloads/  which isn't a maven or ivy
+    repository. -->
+  <info organisation="org.elasticsearch" module="elasticsearch"/>
+</ivy-module>
diff --git a/qa/vagrant/versions b/qa/vagrant/versions
index 654a95a3a25..0062ac97180 100644
--- a/qa/vagrant/versions
+++ b/qa/vagrant/versions
@@ -1 +1 @@
-6.0.0-alpha1-SNAPSHOT
+5.0.0