diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchBackwardsCompatIntegrationTest.java b/src/test/java/org/elasticsearch/test/ElasticsearchBackwardsCompatIntegrationTest.java
index e317687ed77..8a2aa6c839c 100644
--- a/src/test/java/org/elasticsearch/test/ElasticsearchBackwardsCompatIntegrationTest.java
+++ b/src/test/java/org/elasticsearch/test/ElasticsearchBackwardsCompatIntegrationTest.java
@@ -89,6 +89,9 @@ public abstract class ElasticsearchBackwardsCompatIntegrationTest extends Elasti
             throw new IllegalArgumentException("Invalid Backwards tests location path:" + path + " version: " + version);
         }
         File file = new File(path, "elasticsearch-" + version);
+        if (!file.exists()) {
+            throw new IllegalArgumentException("Backwards tests location is missing: " + file.getAbsolutePath());
+        }
         if (!file.isDirectory()) {
             throw new IllegalArgumentException("Backwards tests location is not a directory: " + file.getAbsolutePath());
         }