Tests: Clean up rest test file handling (#21392)

This change simplifies how the rest test runner finds test files and
removes all leniency.  Previously multiple prefixes and suffixes would
be tried, and tests could exist inside or outside of the classpath,
although outside of the classpath never quite worked. Now only classpath
tests are supported, and only one resource prefix is supported,
`/rest-api-spec/tests`.

closes #20240

distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java

@@ -20,12 +20,9 @@
 package org.elasticsearch.test.rest;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class DebClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public DebClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
@@ -33,7 +30,7 @@ public class DebClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java

@@ -24,8 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class IntegTestZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public IntegTestZipClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
@@ -33,7 +31,7 @@ public class IntegTestZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java

@@ -24,8 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class RpmClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public RpmClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
@@ -33,7 +31,7 @@ public class RpmClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java

@@ -24,8 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class TarClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public TarClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
@@ -33,7 +31,7 @@ public class TarClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java

@@ -24,8 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 public class ZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public ZipClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
@@ -33,7 +31,7 @@ public class ZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -25,7 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
 import java.util.List;
 
 public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -35,7 +34,7 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 

modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java

@@ -24,15 +24,13 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class MatrixStatsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public MatrixStatsClientYamlTestSuiteIT(@Name("yaml")ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class IngestCommonClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public IngestCommonClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class IngestCommonClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class LangExpressionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public LangExpressionClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class LangExpressionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class LangMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public LangMustacheClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class LangMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 /** Runs yaml rest tests */
 public class LangPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,7 +33,7 @@ public class LangPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java

@@ -21,18 +21,17 @@ package org.elasticsearch.percolator;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class PercolatorClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public PercolatorClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java

@@ -25,15 +25,13 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class ReindexClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public ReindexClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

@@ -21,11 +21,10 @@ package org.elasticsearch.repositories.url;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -33,7 +32,7 @@ public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java

@@ -27,8 +27,6 @@ import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 //TODO: This is a *temporary* workaround to ensure a timeout does not mask other problems
 @TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
 public class Netty4ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -38,7 +36,7 @@ public class Netty4ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 

plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class IcuClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public IcuClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class IcuClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class KuromojiClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public KuromojiClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class KuromojiClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class PhoneticClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public PhoneticClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class PhoneticClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class SmartCNClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public SmartCNClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class SmartCNClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class StempelClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public StempelClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class StempelClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class UkrainianClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public UkrainianClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class UkrainianClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class DiscoveryAzureClassicClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public DiscoveryAzureClassicClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class DiscoveryAzureClassicClientYamlTestSuiteIT extends ESClientYamlSuit
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/CloudAwsClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class CloudAwsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public CloudAwsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class CloudAwsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java

@@ -21,12 +21,9 @@ package org.elasticsearch.discovery.file;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 /**
  * Integration tests to make sure the file-based discovery plugin works in a cluster.
  */
@@ -37,7 +34,7 @@ public class FileBasedDiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTe
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class DiscoveryGceClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public DiscoveryGceClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class DiscoveryGceClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class IngestAttachmentClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public IngestAttachmentClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class IngestAttachmentClientYamlTestSuiteIT extends ESClientYamlSuiteTest
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public IngestGeoIpClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class IngestUserAgentClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public IngestUserAgentClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class IngestUserAgentClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class JvmExampleClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public JvmExampleClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class JvmExampleClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class MapperMurmur3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public MapperMurmur3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class MapperMurmur3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class MapperSizeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public MapperSizeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class MapperSizeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public RepositoryAzureClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class RepositoryGcsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public RepositoryGcsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class RepositoryGcsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java

@@ -24,8 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class RepositoryHdfsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public RepositoryHdfsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -33,7 +31,7 @@ public class RepositoryHdfsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class RepositoryS3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public RepositoryS3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class RepositoryS3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class StoreSmbClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public StoreSmbClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class StoreSmbClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java

@@ -21,13 +21,10 @@ package org.elasticsearch.backwards;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 @TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
 public class Backwards50ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -36,7 +33,7 @@ public class Backwards50ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 

qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java

@@ -19,14 +19,13 @@
 
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 @TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
 public class MultiClusterSearchYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,13 +34,13 @@ public class MultiClusterSearchYamlTestSuiteIT extends ESClientYamlSuiteTestCase
         return true;
     }
 
-    public MultiClusterSearchYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
+    public MultiClusterSearchYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
-}
 
+}

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -21,13 +21,10 @@ package org.elasticsearch.upgrades;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 @TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
 public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -46,7 +43,7 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 }

qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class SmokeTestIngestDisabledClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public SmokeTestIngestDisabledClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class SmokeTestIngestDisabledClientYamlTestSuiteIT extends ESClientYamlSu
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 

qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class SmokeTestIngestWithAllDepsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public SmokeTestIngestWithAllDepsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class SmokeTestIngestWithAllDepsClientYamlTestSuiteIT extends ESClientYam
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 

qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -21,14 +21,12 @@ package org.elasticsearch.smoketest;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 @TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
 public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -37,7 +35,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class SmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public SmokeTestPluginsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -34,7 +32,7 @@ public class SmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTest
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java

@@ -25,15 +25,13 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class SmokeTestReindexWithPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     public SmokeTestReindexWithPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }

qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java

@@ -25,8 +25,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
-import java.io.IOException;
-
 public class TribeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     // tribe nodes can not handle delete indices requests
@@ -41,13 +39,13 @@ public class TribeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         return true;
     }
 
-    public TribeClientYamlTestSuiteIT(@Name("yaml") final ClientYamlTestCandidate testCandidate) {
+    public TribeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException {
+    public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
-
 }
+

test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java

@@ -37,7 +37,6 @@ import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_ITERATIONS;
 import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_PREFIX;
 import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_TESTMETHOD;
 import static org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.REST_TESTS_BLACKLIST;
-import static org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.REST_TESTS_SPEC;
 import static org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.REST_TESTS_SUITE;
 
 /**
@@ -152,7 +151,7 @@ public class ReproduceInfoPrinter extends RunListener {
         }
 
         public ReproduceErrorMessageBuilder appendClientYamlSuiteProperties() {
-            return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC, REST_TESTS_BLACKLIST);
+            return appendProperties(REST_TESTS_SUITE, REST_TESTS_BLACKLIST);
         }
 
         protected ReproduceErrorMessageBuilder appendProperties(String... properties) {

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java

@@ -19,8 +19,18 @@
 
 package org.elasticsearch.test.rest.yaml;
 
-import com.carrotsearch.randomizedtesting.RandomizedTest;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
+import com.carrotsearch.randomizedtesting.RandomizedTest;
 import org.apache.http.HttpHost;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
@@ -31,6 +41,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
@@ -42,22 +53,6 @@ import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.junit.AfterClass;
 import org.junit.Before;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.FileSystem;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 /**
  * Runs a suite of yaml tests shared with all the official Elasticsearch clients against against an elasticsearch cluster.
  */
@@ -77,15 +72,9 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
      * Property that allows to control whether spec validation is enabled or not (default true).
      */
     private static final String REST_TESTS_VALIDATE_SPEC = "tests.rest.validate_spec";
-    /**
-     * Property that allows to control where the REST spec files need to be loaded from
-     */
-    public static final String REST_TESTS_SPEC = "tests.rest.spec";
 
-    private static final String REST_LOAD_PACKAGED_TESTS = "tests.rest.load_packaged";
+    private static final String TESTS_PATH = "/rest-api-spec/test";
-
+    private static final String SPEC_PATH = "/rest-api-spec/api";
-    private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test";
-    private static final String DEFAULT_SPEC_PATH = "/rest-api-spec/api";
 
     /**
      * This separator pattern matches ',' except it is preceded by a '\'.
@@ -109,20 +98,11 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
     }
 
     @Before
-    public void initAndResetContext() throws IOException {
+    public void initAndResetContext() throws Exception {
         if (restTestExecutionContext == null) {
             assert adminExecutionContext == null;
             assert blacklistPathMatchers == null;
-            String[] specPaths = resolvePathsProperty(REST_TESTS_SPEC, DEFAULT_SPEC_PATH);
+            ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load(SPEC_PATH);
-            ClientYamlSuiteRestSpec restSpec = null;
-            FileSystem fileSystem = getFileSystem();
-            // don't make a try-with, getFileSystem returns null
-            // ... and you can't close() the default filesystem
-            try {
-                restSpec = ClientYamlSuiteRestSpec.parseFrom(fileSystem, DEFAULT_SPEC_PATH, specPaths);
-            } finally {
-                IOUtils.close(fileSystem);
-            }
             validateSpec(restSpec);
             List<HttpHost> hosts = getClusterHosts();
             RestClient restClient = client();
@@ -188,41 +168,58 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         super.afterIfFailed(errors);
     }
 
-    public static Iterable<Object[]> createParameters() throws IOException {
+    public static Iterable<Object[]> createParameters() throws Exception {
-        List<ClientYamlTestCandidate> restTestCandidates = collectTestCandidates();
+        String[] paths = resolvePathsProperty(REST_TESTS_SUITE, ""); // default to all tests under the test root
-        List<Object[]> objects = new ArrayList<>();
+        List<Object[]> tests = new ArrayList<>();
-        for (ClientYamlTestCandidate restTestCandidate : restTestCandidates) {
+        Map<String, Set<Path>> yamlSuites = loadYamlSuites(paths);
-            objects.add(new Object[]{restTestCandidate});
+        // yaml suites are grouped by directory (effectively by api)
-        }
+        for (String api : yamlSuites.keySet()) {
-        return objects;
+            List<Path> yamlFiles = new ArrayList<>(yamlSuites.get(api));
-    }
+            for (Path yamlFile : yamlFiles) {
-
+                ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(api, yamlFile);
-    private static List<ClientYamlTestCandidate> collectTestCandidates() throws IOException {
+                for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) {
-        List<ClientYamlTestCandidate> testCandidates = new ArrayList<>();
+                    tests.add(new Object[]{ new ClientYamlTestCandidate(restTestSuite, testSection) });
-        FileSystem fileSystem = getFileSystem();
-        // don't make a try-with, getFileSystem returns null
-        // ... and you can't close() the default filesystem
-        try {
-            String[] paths = resolvePathsProperty(REST_TESTS_SUITE, DEFAULT_TESTS_PATH);
-            Map<String, Set<Path>> yamlSuites = FileUtils.findYamlSuites(fileSystem, DEFAULT_TESTS_PATH, paths);
-            //yaml suites are grouped by directory (effectively by api)
-            for (String api : yamlSuites.keySet()) {
-                List<Path> yamlFiles = new ArrayList<>(yamlSuites.get(api));
-                for (Path yamlFile : yamlFiles) {
-                    ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(api, yamlFile);
-                    for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) {
-                        testCandidates.add(new ClientYamlTestCandidate(restTestSuite, testSection));
-                    }
                 }
             }
-        } finally {
-            IOUtils.close(fileSystem);
         }
 
         //sort the candidates so they will always be in the same order before being shuffled, for repeatability
-        Collections.sort(testCandidates, (o1, o2) -> o1.getTestPath().compareTo(o2.getTestPath()));
+        Collections.sort(tests,
+            (o1, o2) -> ((ClientYamlTestCandidate)o1[0]).getTestPath().compareTo(((ClientYamlTestCandidate)o2[0]).getTestPath()));
+        return tests;
+    }
 
-        return testCandidates;
+    /** Find all yaml suites that match the given list of paths from the root test path. */
+    // pkg private for tests
+    static Map<String, Set<Path>> loadYamlSuites(String... paths) throws Exception {
+        Map<String, Set<Path>> files = new HashMap<>();
+        Path root = PathUtils.get(ESClientYamlSuiteTestCase.class.getResource(TESTS_PATH).toURI());
+        for (String strPath : paths) {
+            Path path = root.resolve(strPath);
+            if (Files.isDirectory(path)) {
+                Files.walk(path).forEach(file -> {
+                    if (file.toString().endsWith(".yaml")) {
+                        addYamlSuite(root, file, files);
+                    }
+                });
+            } else {
+                path = root.resolve(strPath + ".yaml");
+                assert Files.exists(path);
+                addYamlSuite(root, path, files);
+            }
+        }
+        return files;
+    }
+
+    /** Add a single suite file to the set of suites. */
+    private static void addYamlSuite(Path root, Path file, Map<String, Set<Path>> files) {
+        String groupName = root.relativize(file.getParent()).toString();
+        Set<Path> filesSet = files.get(groupName);
+        if (filesSet == null) {
+            filesSet = new HashSet<>();
+            files.put(groupName, filesSet);
+        }
+        filesSet.add(file);
     }
 
     private static String[] resolvePathsProperty(String propertyName, String defaultValue) {
@@ -234,34 +231,6 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         }
     }
 
-    /**
-     * Returns a new FileSystem to read REST resources, or null if they
-     * are available from classpath.
-     */
-    @SuppressForbidden(reason = "proper use of URL, hack around a JDK bug")
-    protected static FileSystem getFileSystem() throws IOException {
-        // REST suite handling is currently complicated, with lots of filtering and so on
-        // For now, to work embedded in a jar, return a ZipFileSystem over the jar contents.
-        URL codeLocation = FileUtils.class.getProtectionDomain().getCodeSource().getLocation();
-        boolean loadPackaged = RandomizedTest.systemPropertyAsBoolean(REST_LOAD_PACKAGED_TESTS, true);
-        if (codeLocation.getFile().endsWith(".jar") && loadPackaged) {
-            try {
-                // hack around a bug in the zipfilesystem implementation before java 9,
-                // its checkWritable was incorrect and it won't work without write permissions.
-                // if we add the permission, it will open jars r/w, which is too scary! so copy to a safe r-w location.
-                Path tmp = Files.createTempFile(null, ".jar");
-                try (InputStream in = FileSystemUtils.openFileURLStream(codeLocation)) {
-                    Files.copy(in, tmp, StandardCopyOption.REPLACE_EXISTING);
-                }
-                return FileSystems.newFileSystem(new URI("jar:" + tmp.toUri()), Collections.emptyMap());
-            } catch (URISyntaxException e) {
-                throw new IOException("couldn't open zipfilesystem: ", e);
-            }
-        } else {
-            return null;
-        }
-    }
-
     protected ClientYamlTestExecutionContext getAdminExecutionContext() {
         return adminExecutionContext;
     }

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/FileUtils.java

@@ -1,170 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.PathUtils;
-
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.DirectoryStream;
-import java.nio.file.FileSystem;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.NotDirectoryException;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public final class FileUtils {
-
-    private static final String YAML_SUFFIX = ".yaml";
-    private static final String JSON_SUFFIX = ".json";
-
-    private FileUtils() {
-
-    }
-
-    /**
-     * Returns the json files found within the directory provided as argument.
-     * Files are looked up in the classpath, or optionally from {@code fileSystem} if its not null.
-     */
-    public static Set<Path> findJsonSpec(FileSystem fileSystem, String optionalPathPrefix, String path) throws IOException {
-        Path dir = resolveFile(fileSystem, optionalPathPrefix, path, null);
-
-        if (!Files.isDirectory(dir)) {
-            throw new NotDirectoryException(path);
-        }
-
-        Set<Path> jsonFiles = new HashSet<>();
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
-            for (Path item : stream) {
-                if (item.toString().endsWith(JSON_SUFFIX)) {
-                    jsonFiles.add(item);
-                }
-            }
-        }
-
-        if (jsonFiles.isEmpty()) {
-            throw new NoSuchFileException(path, null, "no json files found");
-        }
-
-        return jsonFiles;
-    }
-
-    /**
-     * Returns the yaml files found within the paths provided.
-     * Each input path can either be a single file (the .yaml suffix is optional) or a directory.
-     * Each path is looked up in the classpath, or optionally from {@code fileSystem} if its not null.
-     */
-    public static Map<String, Set<Path>> findYamlSuites(FileSystem fileSystem, String optionalPathPrefix, final String... paths)
-            throws IOException {
-        Map<String, Set<Path>> yamlSuites = new HashMap<>();
-        for (String path : paths) {
-            collectFiles(resolveFile(fileSystem, optionalPathPrefix, path, YAML_SUFFIX), YAML_SUFFIX, yamlSuites);
-        }
-        return yamlSuites;
-    }
-
-    private static Path resolveFile(FileSystem fileSystem, String optionalPathPrefix, String path, String optionalFileSuffix)
-            throws IOException {
-        if (fileSystem != null) {
-            Path file = findFile(fileSystem, path, optionalFileSuffix);
-            if (!lenientExists(file)) {
-                // try with optional prefix: /rest-api-spec/test (or /rest-api-spec/api) is optional
-                String newPath = optionalPathPrefix + "/" + path;
-                file = findFile(fileSystem, newPath, optionalFileSuffix);
-                if (!lenientExists(file)) {
-                    throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: "
-                            + optionalFileSuffix);
-                }
-            }
-            return file;
-        } else {
-            //try within classpath
-            URL resource = findResource(path, optionalFileSuffix);
-            if (resource == null) {
-                //try within classpath with optional prefix: /rest-api-spec/test (or /rest-api-spec/api) is optional
-                String newPath = optionalPathPrefix + "/" + path;
-                resource = findResource(newPath, optionalFileSuffix);
-                if (resource == null) {
-                    throw new NoSuchFileException(path);
-                }
-            }
-            try {
-                return PathUtils.get(resource.toURI());
-            } catch (Exception e) {
-                // some filesystems have REALLY useless exceptions here.
-                // ZipFileSystem I am looking at you.
-                throw new RuntimeException("couldn't retrieve URL: " + resource, e);
-            }
-        }
-    }
-
-    private static URL findResource(String path, String optionalFileSuffix) {
-        URL resource = FileUtils.class.getResource(path);
-        if (resource == null) {
-            //if not found we append the file suffix to the path (as it is optional)
-            if (Strings.hasLength(optionalFileSuffix) && !path.endsWith(optionalFileSuffix)) {
-                resource = FileUtils.class.getResource(path + optionalFileSuffix);
-            }
-        }
-        return resource;
-    }
-    
-    // used because this test "guesses" from like 4 different places from the filesystem!
-    private static boolean lenientExists(Path file) {
-        boolean exists = false; 
-        try {
-            exists = Files.exists(file);
-        } catch (SecurityException ok) {}
-        return exists;
-    }
-
-    private static Path findFile(FileSystem fileSystem, String path, String optionalFileSuffix) {
-        Path file = fileSystem.getPath(path);
-        if (!lenientExists(file)) {
-            file = fileSystem.getPath(path + optionalFileSuffix);
-        }
-        return file;
-    }
-
-    private static void collectFiles(final Path dir, final String fileSuffix, final Map<String, Set<Path>> files) throws IOException {
-        Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
-            @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                if (file.toString().endsWith(fileSuffix)) {
-                    String groupName = dir.relativize(file.getParent()).toString();
-                    Set<Path> filesSet = files.get(groupName);
-                    if (filesSet == null) {
-                        filesSet = new HashSet<>();
-                        files.put(groupName, filesSet);
-                    }
-                    filesSet.add(file);
-                }
-                return FileVisitResult.CONTINUE;
-            }
-        });
-    }
-}

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java

@@ -18,14 +18,9 @@
  */
 package org.elasticsearch.test.rest.yaml.restspec;
 
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.yaml.FileUtils;
-
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.FileSystem;
+import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
@@ -33,6 +28,12 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
+
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 
 /**
  * Holds the specification used to turn {@code do} actions in the YAML suite into REST api calls.
@@ -41,8 +42,7 @@ public class ClientYamlSuiteRestSpec {
     private final Set<String> globalParameters = new HashSet<>();
     private final Map<String, ClientYamlSuiteRestApi> restApiMap = new HashMap<>();
 
-    private ClientYamlSuiteRestSpec() {
+    private ClientYamlSuiteRestSpec() {}
-    }
 
     private void addApi(ClientYamlSuiteRestApi restApi) {
         ClientYamlSuiteRestApi previous = restApiMap.putIfAbsent(restApi.getName(), restApi);
@@ -78,51 +78,58 @@ public class ClientYamlSuiteRestSpec {
     /**
      * Parses the complete set of REST spec available under the provided directories
      */
-    public static ClientYamlSuiteRestSpec parseFrom(FileSystem fileSystem, String optionalPathPrefix, String... paths) throws IOException {
+    public static ClientYamlSuiteRestSpec load(String classpathPrefix) throws Exception {
+        Path dir = PathUtils.get(ClientYamlSuiteRestSpec.class.getResource(classpathPrefix).toURI());
         ClientYamlSuiteRestSpec restSpec = new ClientYamlSuiteRestSpec();
         ClientYamlSuiteRestApiParser restApiParser = new ClientYamlSuiteRestApiParser();
-        for (String path : paths) {
+        try (Stream<Path> stream = Files.walk(dir)) {
-            for (Path jsonFile : FileUtils.findJsonSpec(fileSystem, optionalPathPrefix, path)) {
+            stream.forEach(item -> {
-                try (InputStream stream = Files.newInputStream(jsonFile)) {
+                if (item.toString().endsWith(".json")) {
-                    String filename = jsonFile.getFileName().toString();
+                    parseSpecFile(restApiParser, item, restSpec);
-                    try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, stream)) {
-                        if (filename.equals("_common.json")) {
-                            String currentFieldName = null;
-                            while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-                                if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
-                                    currentFieldName = parser.currentName();
-                                } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
-                                        && "params".equals(currentFieldName)) {
-                                    while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
-                                        String param = parser.currentName();
-                                        if (restSpec.globalParameters.contains(param)) {
-                                            throw new IllegalArgumentException("Found duplicate global param [" + param + "]");
-                                        }
-                                        restSpec.globalParameters.add(param);
-                                        parser.nextToken();
-                                        if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
-                                            throw new IllegalArgumentException("Expected params field in rest api definition to " +
-                                                    "contain an object");
-                                        }
-                                        parser.skipChildren();
-                                    }
-                                }
-                            }
-                        } else {
-                            ClientYamlSuiteRestApi restApi = restApiParser.parse(jsonFile.toString(), parser);
-                            String expectedApiName = filename.substring(0, filename.lastIndexOf('.'));
-                            if (restApi.getName().equals(expectedApiName) == false) {
-                                throw new IllegalArgumentException("found api [" + restApi.getName() + "] in [" + jsonFile.toString() +
-                                        "]. " + "Each api is expected to have the same name as the file that defines it.");
-                            }
-                            restSpec.addApi(restApi);
-                        }
-                    }
-                } catch (Exception ex) {
-                    throw new IOException("Can't parse rest spec file: [" + jsonFile + "]", ex);
                 }
-            }
+            });
         }
         return restSpec;
     }
+
+    private static void parseSpecFile(ClientYamlSuiteRestApiParser restApiParser, Path jsonFile, ClientYamlSuiteRestSpec restSpec) {
+        try (InputStream stream = Files.newInputStream(jsonFile)) {
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, stream)) {
+                String filename = jsonFile.getFileName().toString();
+                if (filename.equals("_common.json")) {
+                    String currentFieldName = null;
+                    while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                        if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
+                            && "params".equals(currentFieldName)) {
+                            while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                                String param = parser.currentName();
+                                if (restSpec.globalParameters.contains(param)) {
+                                    throw new IllegalArgumentException("Found duplicate global param [" + param + "]");
+                                }
+                                restSpec.globalParameters.add(param);
+                                parser.nextToken();
+                                if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+                                    throw new IllegalArgumentException("Expected params field in rest api definition to " +
+                                        "contain an object");
+                                }
+                                parser.skipChildren();
+                            }
+                        }
+                    }
+                } else {
+                    ClientYamlSuiteRestApi restApi = restApiParser.parse(jsonFile.toString(), parser);
+                    String expectedApiName = filename.substring(0, filename.lastIndexOf('.'));
+                    if (restApi.getName().equals(expectedApiName) == false) {
+                        throw new IllegalArgumentException("found api [" + restApi.getName() + "] in [" + jsonFile.toString() + "]. " +
+                            "Each api is expected to have the same name as the file that defines it.");
+                    }
+                    restSpec.addApi(restApi);
+                }
+            }
+        } catch (IOException ex) {
+            throw new UncheckedIOException("Can't parse rest spec file: [" + jsonFile + "]", ex);
+        }
+    }
 }

test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java

@@ -18,44 +18,43 @@
  */
 package org.elasticsearch.test.rest.yaml;
 
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.yaml.FileUtils;
-
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
 import java.util.Set;
 
+import org.elasticsearch.test.ESTestCase;
+
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.greaterThan;
 
-public class FileUtilsTests extends ESTestCase {
+public class ESClientYamlSuiteTestCaseTests extends ESTestCase {
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/20240")
-    public void testLoadSingleYamlSuite() throws Exception {
-        Map<String,Set<Path>> yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "/rest-api-spec/test/suite1/10_basic");
-        assertSingleFile(yamlSuites, "suite1", "10_basic.yaml");
 
-        //the path prefix is optional
+    public void testLoadAllYamlSuites() throws Exception {
-        yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1/10_basic.yaml");
+        Map<String,Set<Path>> yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("");
+        assertEquals(2, yamlSuites.size());
+    }
+
+    public void testLoadSingleYamlSuite() throws Exception {
+        Map<String,Set<Path>> yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite1/10_basic");
         assertSingleFile(yamlSuites, "suite1", "10_basic.yaml");
 
         //extension .yaml is optional
-        yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1/10_basic");
+        yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite1/10_basic");
         assertSingleFile(yamlSuites, "suite1", "10_basic.yaml");
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/20240")
     public void testLoadMultipleYamlSuites() throws Exception {
         //single directory
-        Map<String,Set<Path>> yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1");
+        Map<String,Set<Path>> yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite1");
         assertThat(yamlSuites, notNullValue());
         assertThat(yamlSuites.size(), equalTo(1));
         assertThat(yamlSuites.containsKey("suite1"), equalTo(true));
         assertThat(yamlSuites.get("suite1").size(), greaterThan(1));
 
         //multiple directories
-        yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite1", "suite2");
+        yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite1", "suite2");
         assertThat(yamlSuites, notNullValue());
         assertThat(yamlSuites.size(), equalTo(2));
         assertThat(yamlSuites.containsKey("suite1"), equalTo(true));
@@ -64,7 +63,7 @@ public class FileUtilsTests extends ESTestCase {
         assertEquals(2, yamlSuites.get("suite2").size());
 
         //multiple paths, which can be both directories or yaml test suites (with optional file extension)
-        yamlSuites = FileUtils.findYamlSuites(null, "/rest-api-spec/test", "suite2/10_basic", "suite1");
+        yamlSuites = ESClientYamlSuiteTestCase.loadYamlSuites("suite2/10_basic", "suite1");
         assertThat(yamlSuites, notNullValue());
         assertThat(yamlSuites.size(), equalTo(2));
         assertThat(yamlSuites.containsKey("suite2"), equalTo(true));
@@ -77,21 +76,6 @@ public class FileUtilsTests extends ESTestCase {
         Path dir = createTempDir();
         Path file = dir.resolve("test_loading.yaml");
         Files.createFile(file);
-
-        //load from directory outside of the classpath
-        yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test", dir.toAbsolutePath().toString());
-        assertThat(yamlSuites, notNullValue());
-        assertThat(yamlSuites.size(), equalTo(1));
-        assertThat(yamlSuites.containsKey(dir.getFileName().toString()), equalTo(true));
-        assertSingleFile(yamlSuites.get(dir.getFileName().toString()), dir.getFileName().toString(), file.getFileName().toString());
-
-        //load from external file (optional extension)
-        yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test",
-                dir.resolve("test_loading").toAbsolutePath().toString());
-        assertThat(yamlSuites, notNullValue());
-        assertThat(yamlSuites.size(), equalTo(1));
-        assertThat(yamlSuites.containsKey(dir.getFileName().toString()), equalTo(true));
-        assertSingleFile(yamlSuites.get(dir.getFileName().toString()), dir.getFileName().toString(), file.getFileName().toString());
     }
 
     private static void assertSingleFile(Map<String, Set<Path>> yamlSuites, String dirName, String fileName) {