honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for `teardown` section in REST tests 

This commits adds support for a `teardown` section that can be defined in REST tests to
clean up any items that may have been created by the test and are not cleaned up by
deletion of indices and templates.
This commit is contained in:
jaymode 2016-06-30 11:33:29 -04:00
parent 0732004ae8
commit 983a64c833
10 changed files with 301 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
View File

@ -20,6 +20,7 @@ Test file structure
A YAML test file consists of:
* an optional `setup` section, followed by
* an optional `teardown` section, followed by
* one or more test sections
For instance:
@ -28,6 +29,10 @@ For instance:
- do: ....
- do: ....
---
teardown:
- do: ....
---
"First test":
- do: ...
@ -42,6 +47,11 @@ For instance:
A `setup` section contains a list of commands to run before each test
section in order to setup the same environment for each test section.
A `teardown` section contains a list of commands to run after each test
section in order to setup the same environment for each test section. This
may be needed for modifications made by the testthat are not cleared by the
deletion of indices and templates.
A test section represents an independent test, containing multiple `do`
statements and assertions. The contents of a test section must be run in
order, but individual test sections may be run in any order, as follows:
@ -49,9 +59,8 @@ order, but individual test sections may be run in any order, as follows:
1. run `setup` (if any)
2. reset the `response` var and the `stash` (see below)
2. run test contents
3. run teardown
The `teardown` should delete all indices and all templates.
3. run `teardown` (if any)
4. delete all indices and all templates
Dot notation:
-------------

15
test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
View File

@ -359,6 +359,9 @@ public abstract class ESRestTestCase extends ESTestCase {
//skip test if the whole suite (yaml file) is disabled
assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getSetupSection().getSkipSection()),
testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
//skip test if the whole suite (yaml file) is disabled
assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getTeardownSection().getSkipSection()),
testCandidate.getTeardownSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
//skip test if test section is disabled
assumeFalse(buildSkipMessage(testCandidate.getTestPath(), testCandidate.getTestSection().getSkipSection()),
testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
@ -391,8 +394,16 @@ public abstract class ESRestTestCase extends ESTestCase {
restTestExecutionContext.clear();
for (ExecutableSection executableSection : testCandidate.getTestSection().getExecutableSections()) {
executableSection.execute(restTestExecutionContext);
try {
for (ExecutableSection executableSection : testCandidate.getTestSection().getExecutableSections()) {
executableSection.execute(restTestExecutionContext);
}
} finally {
logger.debug("start teardown test [{}]", testCandidate.getTestPath());
for (DoSection doSection : testCandidate.getTeardownSection().getDoSections()) {
doSection.execute(restTestExecutionContext);
}
logger.debug("end teardown test [{}]", testCandidate.getTestPath());
}
}

5
test/framework/src/main/java/org/elasticsearch/test/rest/RestTestCandidate.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.test.rest;
import org.elasticsearch.test.rest.section.RestTestSuite;
import org.elasticsearch.test.rest.section.SetupSection;
import org.elasticsearch.test.rest.section.TeardownSection;
import org.elasticsearch.test.rest.section.TestSection;
/**
@ -56,6 +57,10 @@ public class RestTestCandidate {
return restTestSuite.getSetupSection();
}
public TeardownSection getTeardownSection() {
return restTestSuite.getTeardownSection();
}
public TestSection getTestSection() {
return testSection;
}

16
test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
View File

@ -25,11 +25,13 @@ import java.util.Map;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.test.rest.section.DoSection;
import org.elasticsearch.test.rest.section.ExecutableSection;
import org.elasticsearch.test.rest.section.ResponseBodyAssertion;
import org.elasticsearch.test.rest.section.SetupSection;
import org.elasticsearch.test.rest.section.SkipSection;
import org.elasticsearch.test.rest.section.TeardownSection;
import org.elasticsearch.test.rest.section.TestSection;
/**
@ -39,6 +41,7 @@ import org.elasticsearch.test.rest.section.TestSection;
public class RestTestSuiteParseContext {
private static final SetupSectionParser SETUP_SECTION_PARSER = new SetupSectionParser();
private static final TeardownSectionParser TEARDOWN_SECTION_PARSER = new TeardownSectionParser();
private static final RestTestSectionParser TEST_SECTION_PARSER = new RestTestSectionParser();
private static final SkipSectionParser SKIP_SECTION_PARSER = new SkipSectionParser();
private static final DoSectionParser DO_SECTION_PARSER = new DoSectionParser();
@ -93,6 +96,19 @@ public class RestTestSuiteParseContext {
return SetupSection.EMPTY;
}
public TeardownSection parseTeardownSection() throws IOException, RestTestParseException {
advanceToFieldName();
if ("teardown".equals(parser.currentName())) {
parser.nextToken();
TeardownSection teardownSection = TEARDOWN_SECTION_PARSER.parse(this);
parser.nextToken();
return teardownSection;
}
return TeardownSection.EMPTY;
}
public TestSection parseTestSection() throws IOException, RestTestParseException {
return TEST_SECTION_PARSER.parse(this);
}

2
test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.test.rest.parser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.yaml.YamlXContent;
import org.elasticsearch.test.rest.section.RestTestSuite;
import org.elasticsearch.test.rest.section.TeardownSection;
import org.elasticsearch.test.rest.section.TestSection;
import java.io.IOException;
@ -75,6 +76,7 @@ public class RestTestSuiteParser implements RestTestFragmentParser<RestTestSuite
RestTestSuite restTestSuite = new RestTestSuite(parseContext.getApi(), parseContext.getSuiteName());
restTestSuite.setSetupSection(parseContext.parseSetupSection());
restTestSuite.setTeardownSection(parseContext.parseTeardownSection());
while(true) {
//the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed

53
test/framework/src/main/java/org/elasticsearch/test/rest/parser/TeardownSectionParser.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.rest.parser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.rest.section.TeardownSection;
import java.io.IOException;
/**
* Parser for teardown section
*/
public class TeardownSectionParser implements RestTestFragmentParser<TeardownSection> {
@Override
public TeardownSection parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
XContentParser parser = parseContext.parser();
TeardownSection teardownSection = new TeardownSection();
teardownSection.setSkipSection(parseContext.parseSkipSection());
while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
parseContext.advanceToFieldName();
if (!"do".equals(parser.currentName())) {
throw new RestTestParseException("section [" + parser.currentName() + "] not supported within teardown section");
}
parser.nextToken();
teardownSection.addDoSection(parseContext.parseDoSection());
parser.nextToken();
}
parser.nextToken();
return teardownSection;
}
}

9
test/framework/src/main/java/org/elasticsearch/test/rest/section/RestTestSuite.java
View File

@ -33,6 +33,7 @@ public class RestTestSuite {
private final String name;
private SetupSection setupSection;
private TeardownSection teardownSection;
private Set<TestSection> testSections = new TreeSet<>();
@ -61,6 +62,14 @@ public class RestTestSuite {
this.setupSection = setupSection;
}
public TeardownSection getTeardownSection() {
return teardownSection;
}
public void setTeardownSection(TeardownSection teardownSection) {
this.teardownSection = teardownSection;
}
/**
* Adds a {@link org.elasticsearch.test.rest.section.TestSection} to the REST suite
* @return true if the test section was not already present, false otherwise

56
test/framework/src/main/java/org/elasticsearch/test/rest/section/TeardownSection.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.rest.section;
import java.util.ArrayList;
import java.util.List;
public class TeardownSection {
public static final TeardownSection EMPTY;
static {
EMPTY = new TeardownSection();
EMPTY.setSkipSection(SkipSection.EMPTY);
}
private SkipSection skipSection;
private List<DoSection> doSections = new ArrayList<>();
public SkipSection getSkipSection() {
return skipSection;
}
public void setSkipSection(SkipSection skipSection) {
this.skipSection = skipSection;
}
public List<DoSection> getDoSections() {
return doSections;
}
public void addDoSection(DoSection doSection) {
this.doSections.add(doSection);
}
public boolean isEmpty() {
return EMPTY.equals(this);
}
}

50
test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java
View File

@ -54,13 +54,30 @@ public class RestTestParserTests extends ESTestCase {
parser.close();
}
public void testParseTestSetupAndSections() throws Exception {
parser = YamlXContent.yamlXContent.createParser(
public void testParseTestSetupTeardownAndSections() throws Exception {
final boolean includeSetup = randomBoolean();
final boolean includeTeardown = randomBoolean();
StringBuilder testSpecBuilder = new StringBuilder();
if (includeSetup) {
testSpecBuilder
.append("---\n" +
"setup:\n" +
" - do:\n" +
" indices.create:\n" +
" index: test_index\n" +
"\n" +
"\n");
}
if (includeTeardown) {
testSpecBuilder
.append("---\n" +
"teardown:\n" +
" - do:\n" +
" indices.delete:\n" +
" index: test_index\n" +
"\n");
}
parser = YamlXContent.yamlXContent.createParser(
testSpecBuilder.toString() +
"---\n" +
"\"Get index mapping\":\n" +
" - do:\n" +
@ -92,12 +109,29 @@ public class RestTestParserTests extends ESTestCase {
assertThat(restTestSuite, notNullValue());
assertThat(restTestSuite.getName(), equalTo("suite"));
assertThat(restTestSuite.getSetupSection(), notNullValue());
assertThat(restTestSuite.getSetupSection().getSkipSection().isEmpty(), equalTo(true));
if (includeSetup) {
assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getSetupSection().getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1));
assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create"));
assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), equalTo("test_index"));
} else {
assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
}
assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1));
assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create"));
assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), equalTo("test_index"));
assertThat(restTestSuite.getTeardownSection(), notNullValue());
if (includeTeardown) {
assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false));
assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true));
assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1));
assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.delete"));
assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
equalTo("test_index"));
} else {
assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(true));
}
assertThat(restTestSuite.getTestSections().size(), equalTo(2));

93
test/framework/src/test/java/org/elasticsearch/test/rest/test/TeardownSectionParserTests.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.rest.test;
import org.elasticsearch.Version;
import org.elasticsearch.common.xcontent.yaml.YamlXContent;
import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext;
import org.elasticsearch.test.rest.parser.TeardownSectionParser;
import org.elasticsearch.test.rest.section.TeardownSection;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
/**
* Unit tests for the teardown section parser
*/
public class TeardownSectionParserTests extends AbstractParserTestCase {
public void testParseTeardownSection() throws Exception {
parser = YamlXContent.yamlXContent.createParser(
" - do:\n" +
" delete:\n" +
" index: foo\n" +
" type: doc\n" +
" id: 1\n" +
" ignore: 404\n" +
" - do:\n" +
" delete2:\n" +
" index: foo\n" +
" type: doc\n" +
" id: 1\n" +
" ignore: 404"
);
TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
TeardownSection section = teardownSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser));
assertThat(section, notNullValue());
assertThat(section.getSkipSection().isEmpty(), equalTo(true));
assertThat(section.getDoSections().size(), equalTo(2));
assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
}
public void testParseWithSkip() throws Exception {
parser = YamlXContent.yamlXContent.createParser(
" - skip:\n" +
" version: \"2.0.0 - 2.3.0\"\n" +
" reason: \"there is a reason\"\n" +
" - do:\n" +
" delete:\n" +
" index: foo\n" +
" type: doc\n" +
" id: 1\n" +
" ignore: 404\n" +
" - do:\n" +
" delete2:\n" +
" index: foo\n" +
" type: doc\n" +
" id: 1\n" +
" ignore: 404"
);
TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
TeardownSection section = teardownSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser));
assertThat(section, notNullValue());
assertThat(section.getSkipSection().isEmpty(), equalTo(false));
assertThat(section.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
assertThat(section.getSkipSection().getUpperVersion(), equalTo(Version.V_2_3_0));
assertThat(section.getSkipSection().getReason(), equalTo("there is a reason"));
assertThat(section.getDoSections().size(), equalTo(2));
assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
}
}