[TEST] allow to disable REST spec validation

With #9629 we introduced REST spec validation, which barfs whenever the REST spec don't follow the defined conventions. That said, we sometimes execute tests against previous branches and tags which have spec that needs fixing but we can't go back and fix them. We now support the `-Dtests.rest.validate_spec` system property that allows to turn off REST spec validation (enabled by default) so that we can still run tests against old branches/tags.

src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java

@@ -25,10 +25,8 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TestGroup;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import com.google.common.collect.Lists;
-
 import org.apache.lucene.util.AbstractRandomizedTest;
 import org.apache.lucene.util.TimeUnits;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -38,6 +36,7 @@ import org.elasticsearch.test.rest.client.RestException;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
 import org.elasticsearch.test.rest.parser.RestTestSuiteParser;
 import org.elasticsearch.test.rest.section.*;
+import org.elasticsearch.test.rest.spec.RestApi;
 import org.elasticsearch.test.rest.spec.RestSpec;
 import org.elasticsearch.test.rest.support.FileUtils;
 import org.junit.AfterClass;
@@ -45,7 +44,6 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
@@ -74,6 +72,10 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
      * e.g. -Dtests.rest.blacklist=get/10_basic/*
      */
     public static final String REST_TESTS_BLACKLIST = "tests.rest.blacklist";
+    /**
+     * Property that allows to control whether spec validation is enabled or not (default true).
+     */
+    public static final String REST_TESTS_VALIDATE_SPEC = "tests.rest.validate_spec";
     /**
      * Property that allows to control where the REST spec files need to be loaded from
      */
@@ -185,9 +187,30 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
     public static void initExecutionContext() throws IOException, RestException {
         String[] specPaths = resolvePathsProperty(REST_TESTS_SPEC, DEFAULT_SPEC_PATH);
         RestSpec restSpec = RestSpec.parseFrom(DEFAULT_SPEC_PATH, specPaths);
+        validateSpec(restSpec);
         restTestExecutionContext = new RestTestExecutionContext(restSpec);
     }
 
+    private static void validateSpec(RestSpec restSpec) {
+        boolean validateSpec = RandomizedTest.systemPropertyAsBoolean(REST_TESTS_VALIDATE_SPEC, true);
+        if (validateSpec) {
+            StringBuilder errorMessage = new StringBuilder();
+            for (RestApi restApi : restSpec.getApis()) {
+                if (restApi.getMethods().contains("GET") && restApi.isBodySupported()) {
+                    if (!restApi.getMethods().contains("POST")) {
+                        errorMessage.append("\n- ").append(restApi.getName()).append(" supports GET with a body but doesn't support POST");
+                    }
+                    if (!restApi.getParams().contains("source")) {
+                        errorMessage.append("\n- ").append(restApi.getName()).append(" supports GET with a body but doesn't support the source query string parameter");
+                    }
+                }
+            }
+            if (errorMessage.length() > 0) {
+                throw new IllegalArgumentException(errorMessage.toString());
+            }
+        }
+    }
+
     @AfterClass
     public static void close() {
         if (restTestExecutionContext != null) {

src/test/java/org/elasticsearch/test/rest/spec/RestSpec.java

@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.Map;
 
 /**
@@ -46,6 +47,10 @@ public class RestSpec {
         return restApiMap.get(api);
     }
 
+    public Collection<RestApi> getApis() {
+        return restApiMap.values();
+    }
+
     /**
      * Parses the complete set of REST spec available under the provided directories
      */
@@ -56,14 +61,6 @@ public class RestSpec {
                 try (InputStream stream = Files.newInputStream(jsonFile)) {
                     XContentParser parser = JsonXContent.jsonXContent.createParser(stream);
                     RestApi restApi = new RestApiParser().parse(parser);
-                    if (restApi.getMethods().contains("GET") && restApi.isBodySupported()) {
-                        if (!restApi.getMethods().contains("POST")) {
-                            throw new IllegalArgumentException(restApi.getName() + " supports GET with a body but doesn't support POST");
-                        }
-                        if (!restApi.getParams().contains("source")) {
-                            throw new IllegalArgumentException(restApi.getName() + " supports GET with a body but doesn't support the source query string parameter");
-                        }
-                    }
                     restSpec.addApi(restApi);
                 } catch (Throwable ex) {
                     throw new IOException("Can't parse rest spec file: [" + jsonFile + "]", ex);