diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
index ecb50ce80b5..8d93301e0c7 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
@@ -67,6 +67,10 @@ class RandomizedTestingTask extends DefaultTask implements ProgressLoggerFactory
     @Input
     String ifNoTests = 'ignore'
 
+    @Optional
+    @Input
+    String onNonEmptyWorkDirectory = 'fail'
+
     TestLoggingConfiguration testLoggingConfig = new TestLoggingConfiguration()
 
     BalancersConfiguration balancersConfig = new BalancersConfiguration(task: this)
@@ -193,6 +197,7 @@ class RandomizedTestingTask extends DefaultTask implements ProgressLoggerFactory
             shuffleOnSlave: shuffleOnSlave,
             leaveTemporary: leaveTemporary,
             ifNoTests: ifNoTests,
+            onNonEmptyWorkDirectory: onNonEmptyWorkDirectory,
             newenvironment: true
         ]
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index e27f041aaab..5e58bcc7878 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -476,6 +476,7 @@ class BuildPlugin implements Plugin<Project> {
             jvm "${project.javaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', 'auto')
             ifNoTests 'fail'
+            onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
 
             // TODO: why are we not passing maxmemory to junit4?
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 1a4c62d2a8a..77157d2f3c8 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -13,7 +13,7 @@ slf4j             = 1.6.2
 jna               = 4.2.2
 
 # test dependencies
-randomizedrunner  = 2.4.0
+randomizedrunner  = 2.5.0
 junit             = 4.11
 httpclient        = 4.5.2
 # When updating httpcore, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index f7654f2dfff..8769c80b84e 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-test-framework-6.5.0-snapshot-d00c5ca.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.randomizedtesting-runner-2.4.0.jar}" {
+grant codeBase "${codebase.randomizedtesting-runner-2.5.0.jar}" {
   // optionally needed for access to private test methods (e.g. beforeClass)
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed to fail tests on uncaught exceptions from other threads