diff --git a/TESTING.asciidoc b/TESTING.asciidoc index bf66dcfe0f2..843ed4b91c1 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -194,6 +194,9 @@ The REST tests support all the options provided by the randomized runner, plus t of the tests providing a sub-folder or even a single yaml file (the default /rest-api-spec/test prefix is optional when files are loaded from classpath) e.g. -Dtests.rest.suite=index,get,create/10_with_id +* `tests.rest.blacklist`: comma separated globs that identify tests that are +blacklisted and need to be skipped +e.g. -Dtests.rest.blacklist=index/*/Index document,get/10_basic/* * `tests.rest.spec`: REST spec path (default /rest-api-spec/api) Note that the REST tests, like all the integration tests, can be run against an external diff --git a/pom.xml b/pom.xml index beb2b3ce532..2ca70f37ca1 100644 --- a/pom.xml +++ b/pom.xml @@ -477,6 +477,7 @@ ${tests.assertion.disabled} ${tests.rest} ${tests.rest.suite} + ${tests.rest.blacklist} ${tests.rest.spec} ${tests.network} ${tests.cluster} diff --git a/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java index f8b571edd7d..e987ca9eb30 100644 --- a/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java +++ b/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java @@ -35,6 +35,7 @@ import org.junit.runner.notification.RunListener; import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_ITERATIONS; import static com.carrotsearch.randomizedtesting.SysGlobals.SYSPROP_TESTMETHOD; import static org.elasticsearch.test.ElasticsearchIntegrationTest.TESTS_CLUSTER; +import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_BLACKLIST; import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_SPEC; import static org.elasticsearch.test.rest.ElasticsearchRestTests.REST_TESTS_SUITE; @@ -142,7 +143,7 @@ public class ReproduceInfoPrinter extends RunListener { } public ReproduceErrorMessageBuilder appendRestTestsProperties() { - return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC); + return appendProperties(REST_TESTS_SUITE, REST_TESTS_SPEC, REST_TESTS_BLACKLIST); } protected ReproduceErrorMessageBuilder appendProperties(String... properties) { diff --git a/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java b/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java index 8d3dee1e4fd..0c94716b5e4 100644 --- a/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java +++ b/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java @@ -37,6 +37,9 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; import java.util.List; import java.util.Map; import java.util.Set; @@ -49,8 +52,23 @@ import java.util.Set; //@ReplicateOnEachVm public class ElasticsearchRestTests extends ElasticsearchIntegrationTest { + /** + * Property that allows to control whether the REST tests need to be run (default) or not (false) + */ public static final String REST_TESTS = "tests.rest"; + /** + * Property that allows to control which REST tests get run. Supports comma separated list of tests + * or directories that contain tests e.g. -Dtests.rest.suite=index,get,create/10_with_id + */ public static final String REST_TESTS_SUITE = "tests.rest.suite"; + /** + * Property that allows to blacklist some of the REST tests based on a comma separated list of globs + * e.g. -Dtests.rest.blacklist=get/10_basic/* + */ + public static final String REST_TESTS_BLACKLIST = "tests.rest.blacklist"; + /** + * Property that allows to control where the REST spec files need to be loaded from + */ public static final String REST_TESTS_SPEC = "tests.rest.spec"; private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test"; @@ -58,6 +76,7 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest { private static final String PATHS_SEPARATOR = ","; + private final PathMatcher[] blacklistPathMatchers; private static RestTestExecutionContext restTestExecutionContext; //private static final int JVM_COUNT = systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_COUNT, 1); @@ -67,6 +86,16 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest { public ElasticsearchRestTests(@Name("yaml") RestTestCandidate testCandidate) { this.testCandidate = testCandidate; + String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null); + if (blacklist != null) { + blacklistPathMatchers = new PathMatcher[blacklist.length]; + int i = 0; + for (String glob : blacklist) { + blacklistPathMatchers[i++] = FileSystems.getDefault().getPathMatcher("glob:" + glob); + } + } else { + blacklistPathMatchers = new PathMatcher[0]; + } } @ParametersFactory @@ -118,7 +147,7 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest { private static String[] resolvePathsProperty(String propertyName, String defaultValue) { String property = System.getProperty(propertyName); if (!Strings.hasLength(property)) { - return new String[]{defaultValue}; + return defaultValue == null ? null : new String[]{defaultValue}; } else { return property.split(PATHS_SEPARATOR); } @@ -143,12 +172,19 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest { @Before public void reset() throws IOException, RestException { + //skip test if it matches one of the blacklist globs + for (PathMatcher blacklistedPathMatcher : blacklistPathMatchers) { + assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher.matches(Paths.get(testCandidate.getTestPath()))); + } + restTestExecutionContext.resetClient(immutableCluster().httpAddresses()); restTestExecutionContext.clear(); - assumeFalse(buildSkipMessage(testCandidate.getSuiteDescription(), testCandidate.getSetupSection().getSkipSection()), + //skip test if the whole suite (yaml file) is disabled + assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getSetupSection().getSkipSection()), testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion())); - assumeFalse(buildSkipMessage(testCandidate.getDescription(), testCandidate.getTestSection().getSkipSection()), + //skip test if test section is disabled + assumeFalse(buildSkipMessage(testCandidate.getTestPath(), testCandidate.getTestSection().getSkipSection()), testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion())); } @@ -166,16 +202,15 @@ public class ElasticsearchRestTests extends ElasticsearchIntegrationTest { public void test() throws IOException { //let's check that there is something to run, otherwise there might be a problem with the test section if (testCandidate.getTestSection().getExecutableSections().size() == 0) { - throw new IllegalArgumentException("No executable sections loaded for [" - + testCandidate.getSuiteDescription() + "/" + testCandidate.getTestSection().getName() + "]"); + throw new IllegalArgumentException("No executable sections loaded for [" + testCandidate.getTestPath() + "]"); } if (!testCandidate.getSetupSection().isEmpty()) { - logger.info("start setup test [{}: {}]", testCandidate.getSuiteDescription(), testCandidate.getTestSection().getName()); + logger.info("start setup test [{}]", testCandidate.getTestPath()); for (DoSection doSection : testCandidate.getSetupSection().getDoSections()) { doSection.execute(restTestExecutionContext); } - logger.info("end setup test [{}: {}]", testCandidate.getSuiteDescription(), testCandidate.getTestSection().getName()); + logger.info("end setup test [{}]", testCandidate.getTestPath()); } restTestExecutionContext.clear(); diff --git a/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java b/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java index 52d03887e2b..e454c396a3d 100644 --- a/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java +++ b/src/test/java/org/elasticsearch/test/rest/RestTestCandidate.java @@ -44,12 +44,12 @@ public class RestTestCandidate { return restTestSuite.getName(); } - public String getSuiteDescription() { - return restTestSuite.getDescription(); + public String getSuitePath() { + return restTestSuite.getPath(); } - public String getDescription() { - return getSuiteDescription() + "/" + testSection.getName(); + public String getTestPath() { + return restTestSuite.getPath() + "/" + testSection.getName(); } public SetupSection getSetupSection() { @@ -62,6 +62,6 @@ public class RestTestCandidate { @Override public String toString() { - return getSuiteDescription() + "/" + testSection.getName(); + return getTestPath(); } } diff --git a/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java b/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java index 56a95871ac1..4c01387ec7b 100644 --- a/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java +++ b/src/test/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java @@ -93,7 +93,7 @@ public class RestTestSuiteParser implements RestTestFragmentParser