diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index bf66dcfe0f2..843ed4b91c1 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -194,6 +194,9 @@ The REST tests support all the options provided by the randomized runner, plus t
of the tests providing a sub-folder or even a single yaml file (the default
/rest-api-spec/test prefix is optional when files are loaded from classpath)
e.g. -Dtests.rest.suite=index,get,create/10_with_id
+* `tests.rest.blacklist`: comma separated globs that identify tests that are
+blacklisted and need to be skipped
+e.g. -Dtests.rest.blacklist=index/*/Index document,get/10_basic/*
* `tests.rest.spec`: REST spec path (default /rest-api-spec/api)
Note that the REST tests, like all the integration tests, can be run against an external
diff --git a/pom.xml b/pom.xml
index beb2b3ce532..2ca70f37ca1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -477,6 +477,7 @@
${tests.assertion.disabled}
${tests.rest}
${tests.rest.suite}
+ ${tests.rest.blacklist}
${tests.rest.spec}
${tests.network}
${tests.cluster}
diff --git a/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
index f8b571edd7d..e987ca9eb30 100644
--- a/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
+++ b/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
@@ -35,6 +35,7 @@ import org.junit.runner.notification.RunListener;
import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_ITERATIONS;
import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_TESTMETHOD;
import static org.elasticsearch.test.ElasticsearchIntegrationTest.TESTS_CLUSTER;
+import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_BLACKLIST;
import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_SPEC;
import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_SUITE;
@@ -142,7 +143,7 @@ public class ReproduceInfoPrinter extends RunListener {
}
public ReproduceErrorMessageBuilder appendRestTestsProperties() {
- return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC);
+ return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC, REST_TESTS_BLACKLIST);
}
protected ReproduceErrorMessageBuilder appendProperties(String... properties) {
diff --git a/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java b/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java
index 8d3dee1e4fd..0c94716b5e4 100644
--- a/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java
+++ b/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java
@@ -37,6 +37,9 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -49,8 +52,23 @@ import java.util.Set;
//@ReplicateOnEachVm
public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
+ /**
+ * Property that allows to control whether the REST tests need to be run (default) or not (false)
+ */
public static final String REST_TESTS = "tests.rest";
+ /**
+ * Property that allows to control which REST tests get run. Supports comma separated list of tests
+ * or directories that contain tests e.g. -Dtests.rest.suite=index,get,create/10_with_id
+ */
public static final String REST_TESTS_SUITE = "tests.rest.suite";
+ /**
+ * Property that allows to blacklist some of the REST tests based on a comma separated list of globs
+ * e.g. -Dtests.rest.blacklist=get/10_basic/*
+ */
+ public static final String REST_TESTS_BLACKLIST = "tests.rest.blacklist";
+ /**
+ * Property that allows to control where the REST spec files need to be loaded from
+ */
public static final String REST_TESTS_SPEC = "tests.rest.spec";
private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test";
@@ -58,6 +76,7 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
private static final String PATHS_SEPARATOR = ",";
+ private final PathMatcher[] blacklistPathMatchers;
private static RestTestExecutionContext restTestExecutionContext;
//private static final int JVM_COUNT = systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_COUNT, 1);
@@ -67,6 +86,16 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
public ElasticsearchRestTests(@Name("yaml") RestTestCandidate testCandidate) {
this.testCandidate = testCandidate;
+ String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null);
+ if (blacklist != null) {
+ blacklistPathMatchers = new PathMatcher[blacklist.length];
+ int i = 0;
+ for (String glob : blacklist) {
+ blacklistPathMatchers[i++] = FileSystems.getDefault().getPathMatcher("glob:" + glob);
+ }
+ } else {
+ blacklistPathMatchers = new PathMatcher[0];
+ }
}
@ParametersFactory
@@ -118,7 +147,7 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
private static String[] resolvePathsProperty(String propertyName, String defaultValue) {
String property = System.getProperty(propertyName);
if (!Strings.hasLength(property)) {
- return new String[]{defaultValue};
+ return defaultValue == null ? null : new String[]{defaultValue};
} else {
return property.split(PATHS_SEPARATOR);
}
@@ -143,12 +172,19 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
@Before
public void reset() throws IOException, RestException {
+ //skip test if it matches one of the blacklist globs
+ for (PathMatcher blacklistedPathMatcher : blacklistPathMatchers) {
+ assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher.matches(Paths.get(testCandidate.getTestPath())));
+ }
+
restTestExecutionContext.resetClient(immutableCluster().httpAddresses());
restTestExecutionContext.clear();
- assumeFalse(buildSkipMessage(testCandidate.getSuiteDescription(), testCandidate.getSetupSection().getSkipSection()),
+ //skip test if the whole suite (yaml file) is disabled
+ assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getSetupSection().getSkipSection()),
testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
- assumeFalse(buildSkipMessage(testCandidate.getDescription(), testCandidate.getTestSection().getSkipSection()),
+ //skip test if test section is disabled
+ assumeFalse(buildSkipMessage(testCandidate.getTestPath(), testCandidate.getTestSection().getSkipSection()),
testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
}
@@ -166,16 +202,15 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
public void test() throws IOException {
//let's check that there is something to run, otherwise there might be a problem with the test section
if (testCandidate.getTestSection().getExecutableSections().size() == 0) {
- throw new IllegalArgumentException("No executable sections loaded for ["
- + testCandidate.getSuiteDescription() + "/" + testCandidate.getTestSection().getName() + "]");
+ throw new IllegalArgumentException("No executable sections loaded for [" + testCandidate.getTestPath() + "]");
}
if (!testCandidate.getSetupSection().isEmpty()) {
- logger.info("start setup test [{}: {}]", testCandidate.getSuiteDescription(), testCandidate.getTestSection().getName());
+ logger.info("start setup test [{}]", testCandidate.getTestPath());
for (DoSection doSection : testCandidate.getSetupSection().getDoSections()) {
doSection.execute(restTestExecutionContext);
}
- logger.info("end setup test [{}: {}]", testCandidate.getSuiteDescription(), testCandidate.getTestSection().getName());
+ logger.info("end setup test [{}]", testCandidate.getTestPath());
}
restTestExecutionContext.clear();
diff --git a/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java b/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java
index 52d03887e2b..e454c396a3d 100644
--- a/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java
+++ b/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java
@@ -44,12 +44,12 @@ public class RestTestCandidate {
return restTestSuite.getName();
}
- public String getSuiteDescription() {
- return restTestSuite.getDescription();
+ public String getSuitePath() {
+ return restTestSuite.getPath();
}
- public String getDescription() {
- return getSuiteDescription() + "/" + testSection.getName();
+ public String getTestPath() {
+ return restTestSuite.getPath() + "/" + testSection.getName();
}
public SetupSection getSetupSection() {
@@ -62,6 +62,6 @@ public class RestTestCandidate {
@Override
public String toString() {
- return getSuiteDescription() + "/" + testSection.getName();
+ return getTestPath();
}
}
diff --git a/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java b/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
index 56a95871ac1..4c01387ec7b 100644
--- a/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
+++ b/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
@@ -93,7 +93,7 @@ public class RestTestSuiteParser implements RestTestFragmentParser