[TEST] Added blacklist to be able to skip specific REST tests

The blacklist can be provided through -Dtests.rest.blacklist and supports a comma separated list of globs
e.g. -Dtests.rest.blacklist=get/10_basic/*,index/*/*

Also added some missing docs and made it clearer that the suite/test descriptions effectively contains their (relative) path (api/yaml_file/test section)

Closes #5881

TESTING.asciidoc

@@ -194,6 +194,9 @@ The REST tests support all the options provided by the randomized runner, plus t
 of the tests providing a sub-folder or even a single yaml file (the default
 /rest-api-spec/test prefix is optional when files are loaded from classpath)
 e.g. -Dtests.rest.suite=index,get,create/10_with_id
+* `tests.rest.blacklist`: comma separated globs that identify tests that are
+blacklisted and need to be skipped
+e.g. -Dtests.rest.blacklist=index/*/Index document,get/10_basic/*
 * `tests.rest.spec`: REST spec path (default /rest-api-spec/api)
 
 Note that the REST tests, like all the integration tests, can be run against an external

pom.xml

@@ -477,6 +477,7 @@
                                 <tests.assertion.disabled>${tests.assertion.disabled}</tests.assertion.disabled>
                                 <tests.rest>${tests.rest}</tests.rest>
                                 <tests.rest.suite>${tests.rest.suite}</tests.rest.suite>
+                                <tests.rest.blacklist>${tests.rest.blacklist}</tests.rest.blacklist>
                                 <tests.rest.spec>${tests.rest.spec}</tests.rest.spec>
                                 <tests.network>${tests.network}</tests.network>
                                 <tests.cluster>${tests.cluster}</tests.cluster>

src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java

@@ -35,6 +35,7 @@ import org.junit.runner.notification.RunListener;
 import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_ITERATIONS;
 import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_TESTMETHOD;
 import static org.elasticsearch.test.ElasticsearchIntegrationTest.TESTS_CLUSTER;
+import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_BLACKLIST;
 import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_SPEC;
 import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_SUITE;
 
@@ -142,7 +143,7 @@ public class ReproduceInfoPrinter extends RunListener {
         }
 
         public ReproduceErrorMessageBuilder appendRestTestsProperties() {
-            return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC);
+            return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC, REST_TESTS_BLACKLIST);
         }
 
         protected ReproduceErrorMessageBuilder appendProperties(String... properties) {

src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java

@@ -37,6 +37,9 @@ import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -49,8 +52,23 @@ import java.util.Set;
 //@ReplicateOnEachVm
 public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
 
+    /**
+     * Property that allows to control whether the REST tests need to be run (default) or not (false)
+     */
     public static final String REST_TESTS = "tests.rest";
+    /**
+     * Property that allows to control which REST tests get run. Supports comma separated list of tests
+     * or directories that contain tests e.g. -Dtests.rest.suite=index,get,create/10_with_id
+     */
     public static final String REST_TESTS_SUITE = "tests.rest.suite";
+    /**
+     * Property that allows to blacklist some of the REST tests based on a comma separated list of globs
+     * e.g. -Dtests.rest.blacklist=get/10_basic/*
+     */
+    public static final String REST_TESTS_BLACKLIST = "tests.rest.blacklist";
+    /**
+     * Property that allows to control where the REST spec files need to be loaded from
+     */
     public static final String REST_TESTS_SPEC = "tests.rest.spec";
 
     private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test";
@@ -58,6 +76,7 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
 
     private static final String PATHS_SEPARATOR = ",";
 
+    private final PathMatcher[] blacklistPathMatchers;
     private static RestTestExecutionContext restTestExecutionContext;
 
     //private static final int JVM_COUNT = systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_COUNT, 1);
@@ -67,6 +86,16 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
 
     public ElasticsearchRestTests(@Name("yaml") RestTestCandidate testCandidate) {
         this.testCandidate = testCandidate;
+        String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null);
+        if (blacklist != null) {
+            blacklistPathMatchers = new PathMatcher[blacklist.length];
+            int i = 0;
+            for (String glob : blacklist) {
+                blacklistPathMatchers[i++] = FileSystems.getDefault().getPathMatcher("glob:" + glob);
+            }
+        } else {
+            blacklistPathMatchers = new PathMatcher[0];
+        }
     }
 
     @ParametersFactory
@@ -118,7 +147,7 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
     private static String[] resolvePathsProperty(String propertyName, String defaultValue) {
         String property = System.getProperty(propertyName);
         if (!Strings.hasLength(property)) {
-            return new String[]{defaultValue};
+            return defaultValue == null ? null : new String[]{defaultValue};
         } else {
             return property.split(PATHS_SEPARATOR);
         }
@@ -143,12 +172,19 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
 
     @Before
     public void reset() throws IOException, RestException {
+        //skip test if it matches one of the blacklist globs
+        for (PathMatcher blacklistedPathMatcher : blacklistPathMatchers) {
+            assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher.matches(Paths.get(testCandidate.getTestPath())));
+        }
+
         restTestExecutionContext.resetClient(immutableCluster().httpAddresses());
         restTestExecutionContext.clear();
 
-        assumeFalse(buildSkipMessage(testCandidate.getSuiteDescription(), testCandidate.getSetupSection().getSkipSection()),
+        //skip test if the whole suite (yaml file) is disabled
+        assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getSetupSection().getSkipSection()),
                 testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
-        assumeFalse(buildSkipMessage(testCandidate.getDescription(), testCandidate.getTestSection().getSkipSection()),
+        //skip test if test section is disabled
+        assumeFalse(buildSkipMessage(testCandidate.getTestPath(), testCandidate.getTestSection().getSkipSection()),
                 testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
     }
 
@@ -166,16 +202,15 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
     public void test() throws IOException {
         //let's check that there is something to run, otherwise there might be a problem with the test section
         if (testCandidate.getTestSection().getExecutableSections().size() == 0) {
-            throw new IllegalArgumentException("No executable sections loaded for ["
-                    + testCandidate.getSuiteDescription() + "/" + testCandidate.getTestSection().getName() + "]");
+            throw new IllegalArgumentException("No executable sections loaded for [" + testCandidate.getTestPath() + "]");
         }
 
         if (!testCandidate.getSetupSection().isEmpty()) {
-            logger.info("start setup test [{}: {}]", testCandidate.getSuiteDescription(), testCandidate.getTestSection().getName());
+            logger.info("start setup test [{}]", testCandidate.getTestPath());
             for (DoSection doSection : testCandidate.getSetupSection().getDoSections()) {
                 doSection.execute(restTestExecutionContext);
             }
-            logger.info("end setup test [{}: {}]", testCandidate.getSuiteDescription(), testCandidate.getTestSection().getName());
+            logger.info("end setup test [{}]", testCandidate.getTestPath());
         }
 
         restTestExecutionContext.clear();

src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java

@@ -44,12 +44,12 @@ public class RestTestCandidate {
         return restTestSuite.getName();
     }
 
-    public String getSuiteDescription() {
-        return restTestSuite.getDescription();
+    public String getSuitePath() {
+        return restTestSuite.getPath();
     }
 
-    public String getDescription() {
-        return getSuiteDescription() + "/" + testSection.getName();
+    public String getTestPath() {
+        return restTestSuite.getPath() + "/" + testSection.getName();
     }
 
     public SetupSection getSetupSection() {
@@ -62,6 +62,6 @@ public class RestTestCandidate {
 
     @Override
     public String toString() {
-        return getSuiteDescription() + "/" + testSection.getName();
+        return getTestPath();
     }
 }

src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java

@@ -93,7 +93,7 @@ public class RestTestSuiteParser implements RestTestFragmentParser<RestTestSuite
 
             TestSection testSection = parseContext.parseTestSection();
             if (!restTestSuite.addTestSection(testSection)) {
-                throw new RestTestParseException("duplicate test section [" + testSection.getName() + "] found in [" + restTestSuite.getDescription() + "]");
+                throw new RestTestParseException("duplicate test section [" + testSection.getName() + "] found in [" + restTestSuite.getPath() + "]");
             }
         }
 

src/test/java/org/elasticsearch/test/rest/section/RestTestSuite.java

@@ -50,9 +50,7 @@ public class RestTestSuite {
         return name;
     }
 
-    //describes the rest test suite (e.g. index/10_with_id)
-    //useful also to reproduce failures
-    public String getDescription() {
+    public String getPath() {
         return api + "/" + name;
     }