Add line number to yaml test failures

Old:
```
   > Throwable #1: java.lang.AssertionError: expected [2xx] status code but api [reindex] returned [400 Bad Request] [{"error":{"root_cause":[{"type":"parsing_exception","reason":"[reindex] failed to parse field [dest]","line":1,"col":25}],"type":"parsing_exception","reason":"[reindex] failed to parse field [dest]","line":1,"col":25,"caused_by":{"type":"illegal_argument_exception","reason":"[dest] unknown field [asdfadf], parser not found"}},"status":400}]
   >    at __randomizedtesting.SeedInfo.seed([9325F8C5C6F227DD:1B71C71F680E4A25]:0)
   >    at org.elasticsearch.test.rest.yaml.section.DoSection.execute(DoSection.java:119)
   >    at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.test(ESClientYamlSuiteTestCase.java:309)
   >    at java.lang.Thread.run(Thread.java:745)
```

New:
```
   > Throwable #1: java.lang.AssertionError: Failure at [reindex/10_basic:12]: expected [2xx] status code but api [reindex] returned [400 Bad Request] [{"error":{"root_cause":[{"type":"parsing_exception","reason":"[reindex] failed to parse field [dest]","line":1,"col":25}],"type":"parsing_exception","reason":"[reindex] failed to parse field [dest]","line":1,"col":25,"caused_by":{"type":"illegal_argument_exception","reason":"[dest] unknown field [asdfadf], parser not found"}},"status":400}]
   >    at __randomizedtesting.SeedInfo.seed([444DEEAF47322306:CC19D175E9CE4EFE]:0)
   >    at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.executeSection(ESClientYamlSuiteTestCase.java:329)
   >    at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.test(ESClientYamlSuiteTestCase.java:309)
   >    at java.lang.Thread.run(Thread.java:745)
   > Caused by: java.lang.AssertionError: expected [2xx] status code but api [reindex] returned [400 Bad Request] [{"error":{"root_cause":[{"type":"parsing_exception","reason":"[reindex] failed to parse field [dest]","line":1,"col":25}],"type":"parsing_exception","reason":"[reindex] failed to parse field [dest]","line":1,"col":25,"caused_by":{"type":"illegal_argument_exception","reason":"[dest] unknown field [asdfadf], parser not found"}},"status":400}]
   >    at org.elasticsearch.test.rest.yaml.section.DoSection.execute(DoSection.java:119)
   >    at org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase.executeSection(ESClientYamlSuiteTestCase.java:325)
   >    ... 37 more
```

Sorry for the longer stack trace, but I wanted to be sure I didn't throw
anything away by accident.
This commit is contained in:
Nik Everett 2016-08-02 16:09:14 -04:00
parent 51bbe2c5c4
commit ca8f666c66
23 changed files with 103 additions and 32 deletions

View File

@ -297,7 +297,7 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
if (!testCandidate.getSetupSection().isEmpty()) {
logger.debug("start setup test [{}]", testCandidate.getTestPath());
for (DoSection doSection : testCandidate.getSetupSection().getDoSections()) {
doSection.execute(restTestExecutionContext);
executeSection(doSection);
}
logger.debug("end setup test [{}]", testCandidate.getTestPath());
}
@ -306,14 +306,31 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
try {
for (ExecutableSection executableSection : testCandidate.getTestSection().getExecutableSections()) {
executableSection.execute(restTestExecutionContext);
executeSection(executableSection);
}
} finally {
logger.debug("start teardown test [{}]", testCandidate.getTestPath());
for (DoSection doSection : testCandidate.getTeardownSection().getDoSections()) {
doSection.execute(restTestExecutionContext);
executeSection(doSection);
}
logger.debug("end teardown test [{}]", testCandidate.getTestPath());
}
}
/**
* Execute an {@link ExecutableSection}, careful to log its place of origin on failure.
*/
private void executeSection(ExecutableSection executableSection) {
try {
executableSection.execute(restTestExecutionContext);
} catch (Exception e) {
throw new RuntimeException(errorMessage(executableSection, e), e);
} catch (AssertionError e) {
throw new AssertionError(errorMessage(executableSection, e), e);
}
}
private String errorMessage(ExecutableSection executableSection, Throwable t) {
return "Failure at [" + testCandidate.getSuitePath() + ":" + executableSection.getLocation().lineNumber + "]: " + t.getMessage();
}
}

View File

@ -41,7 +41,7 @@ public class DoSectionParser implements ClientYamlTestFragmentParser<DoSection>
String currentFieldName = null;
XContentParser.Token token;
DoSection doSection = new DoSection();
DoSection doSection = new DoSection(parseContext.parser().getTokenLocation());
ApiCallSection apiCallSection = null;
Map<String, String> headers = new HashMap<>();

View File

@ -20,6 +20,7 @@
package org.elasticsearch.test.rest.yaml.parser;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.section.GreaterThanEqualToAssertion;
import java.io.IOException;
@ -31,11 +32,12 @@ public class GreaterThanEqualToParser implements ClientYamlTestFragmentParser<Gr
@Override
public GreaterThanEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext)
throws IOException, ClientYamlTestParseException {
XContentLocation location = parseContext.parser().getTokenLocation();
Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
if (! (stringObjectTuple.v2() instanceof Comparable) ) {
throw new ClientYamlTestParseException("gte section can only be used with objects that support natural ordering, found "
+ stringObjectTuple.v2().getClass().getSimpleName());
}
return new GreaterThanEqualToAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
}
}

View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.rest.yaml.parser;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
import java.io.IOException;
@ -30,11 +31,12 @@ public class GreaterThanParser implements ClientYamlTestFragmentParser<GreaterTh
@Override
public GreaterThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
XContentLocation location = parseContext.parser().getTokenLocation();
Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
if (! (stringObjectTuple.v2() instanceof Comparable) ) {
throw new ClientYamlTestParseException("gt section can only be used with objects that support natural ordering, found "
+ stringObjectTuple.v2().getClass().getSimpleName());
}
return new GreaterThanAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
}
}

View File

@ -29,6 +29,6 @@ public class IsFalseParser implements ClientYamlTestFragmentParser<IsFalseAssert
@Override
public IsFalseAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
return new IsFalseAssertion(parseContext.parseField());
return new IsFalseAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField());
}
}

View File

@ -29,6 +29,6 @@ public class IsTrueParser implements ClientYamlTestFragmentParser<IsTrueAssertio
@Override
public IsTrueAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
return new IsTrueAssertion(parseContext.parseField());
return new IsTrueAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField());
}
}

View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.rest.yaml.parser;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
import java.io.IOException;
@ -30,6 +31,7 @@ public class LengthParser implements ClientYamlTestFragmentParser<LengthAssertio
@Override
public LengthAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
XContentLocation location = parseContext.parser().getTokenLocation();
Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
assert stringObjectTuple.v2() != null;
int value;
@ -43,6 +45,6 @@ public class LengthParser implements ClientYamlTestFragmentParser<LengthAssertio
}
}
return new LengthAssertion(stringObjectTuple.v1(), value);
return new LengthAssertion(location, stringObjectTuple.v1(), value);
}
}

View File

@ -20,6 +20,7 @@
package org.elasticsearch.test.rest.yaml.parser;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.section.LessThanOrEqualToAssertion;
import java.io.IOException;
@ -31,11 +32,12 @@ public class LessThanOrEqualToParser implements ClientYamlTestFragmentParser<Les
@Override
public LessThanOrEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
XContentLocation location = parseContext.parser().getTokenLocation();
Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
if (! (stringObjectTuple.v2() instanceof Comparable) ) {
throw new ClientYamlTestParseException("lte section can only be used with objects that support natural ordering, found "
+ stringObjectTuple.v2().getClass().getSimpleName());
}
return new LessThanOrEqualToAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
}
}

View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.rest.yaml.parser;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
import java.io.IOException;
@ -30,11 +31,12 @@ public class LessThanParser implements ClientYamlTestFragmentParser<LessThanAsse
@Override
public LessThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
XContentLocation location = parseContext.parser().getTokenLocation();
Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
if (! (stringObjectTuple.v2() instanceof Comparable) ) {
throw new ClientYamlTestParseException("lt section can only be used with objects that support natural ordering, found "
+ stringObjectTuple.v2().getClass().getSimpleName());
}
return new LessThanAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
}
}

View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.rest.yaml.parser;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
import java.io.IOException;
@ -30,7 +31,8 @@ public class MatchParser implements ClientYamlTestFragmentParser<MatchAssertion>
@Override
public MatchAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
XContentLocation location = parseContext.parser().getTokenLocation();
Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
return new MatchAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
}
}

View File

@ -36,7 +36,7 @@ public class SetSectionParser implements ClientYamlTestFragmentParser<SetSection
String currentFieldName = null;
XContentParser.Token token;
SetSection setSection = new SetSection();
SetSection setSection = new SetSection(parser.getTokenLocation());
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {

View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import java.io.IOException;
@ -27,11 +28,12 @@ import java.util.Map;
* Base class for executable sections that hold assertions
*/
public abstract class Assertion implements ExecutableSection {
private final XContentLocation location;
private final String field;
private final Object expectedValue;
protected Assertion(String field, Object expectedValue) {
protected Assertion(XContentLocation location, String field, Object expectedValue) {
this.location = location;
this.field = field;
this.expectedValue = expectedValue;
}
@ -64,6 +66,11 @@ public abstract class Assertion implements ExecutableSection {
return executionContext.response(field);
}
@Override
public XContentLocation getLocation() {
return location;
}
@Override
public final void execute(ClientYamlTestExecutionContext executionContext) throws IOException {
doAssert(getActualValue(executionContext), resolveExpectedValue(executionContext));

View File

@ -22,6 +22,7 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
import org.elasticsearch.test.rest.yaml.ClientYamlTestResponseException;
@ -59,9 +60,14 @@ public class DoSection implements ExecutableSection {
private static final ESLogger logger = Loggers.getLogger(DoSection.class);
private final XContentLocation location;
private String catchParam;
private ApiCallSection apiCallSection;
public DoSection(XContentLocation location) {
this.location = location;
}
public String getCatch() {
return catchParam;
}
@ -78,6 +84,11 @@ public class DoSection implements ExecutableSection {
this.apiCallSection = apiCallSection;
}
@Override
public XContentLocation getLocation() {
return location;
}
@Override
public void execute(ClientYamlTestExecutionContext executionContext) throws IOException {

View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import java.io.IOException;
@ -26,6 +27,10 @@ import java.io.IOException;
* Represents a test fragment that can be executed (e.g. api call, assertion)
*/
public interface ExecutableSection {
/**
* Get the location in the test that this was defined.
*/
XContentLocation getLocation();
/**
* Executes the section passing in the execution context

View File

@ -20,6 +20,7 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.instanceOf;
@ -35,8 +36,8 @@ public class GreaterThanAssertion extends Assertion {
private static final ESLogger logger = Loggers.getLogger(GreaterThanAssertion.class);
public GreaterThanAssertion(String field, Object expectedValue) {
super(field, expectedValue);
public GreaterThanAssertion(XContentLocation location, String field, Object expectedValue) {
super(location, field, expectedValue);
}
@Override

View File

@ -21,6 +21,7 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.instanceOf;
@ -36,8 +37,8 @@ public class GreaterThanEqualToAssertion extends Assertion {
private static final ESLogger logger = Loggers.getLogger(GreaterThanEqualToAssertion.class);
public GreaterThanEqualToAssertion(String field, Object expectedValue) {
super(field, expectedValue);
public GreaterThanEqualToAssertion(XContentLocation location, String field, Object expectedValue) {
super(location, field, expectedValue);
}
@Override

View File

@ -20,6 +20,7 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.equalTo;
@ -36,8 +37,8 @@ public class IsFalseAssertion extends Assertion {
private static final ESLogger logger = Loggers.getLogger(IsFalseAssertion.class);
public IsFalseAssertion(String field) {
super(field, false);
public IsFalseAssertion(XContentLocation location, String field) {
super(location, field, false);
}
@Override

View File

@ -20,6 +20,7 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.equalToIgnoringCase;
@ -37,8 +38,8 @@ public class IsTrueAssertion extends Assertion {
private static final ESLogger logger = Loggers.getLogger(IsTrueAssertion.class);
public IsTrueAssertion(String field) {
super(field, true);
public IsTrueAssertion(XContentLocation location, String field) {
super(location, field, true);
}
@Override

View File

@ -20,6 +20,7 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import java.util.List;
import java.util.Map;
@ -37,8 +38,8 @@ public class LengthAssertion extends Assertion {
private static final ESLogger logger = Loggers.getLogger(LengthAssertion.class);
public LengthAssertion(String field, Object expectedValue) {
super(field, expectedValue);
public LengthAssertion(XContentLocation location, String field, Object expectedValue) {
super(location, field, expectedValue);
}
@Override

View File

@ -20,6 +20,7 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.lessThan;
@ -36,8 +37,8 @@ public class LessThanAssertion extends Assertion {
private static final ESLogger logger = Loggers.getLogger(LessThanAssertion.class);
public LessThanAssertion(String field, Object expectedValue) {
super(field, expectedValue);
public LessThanAssertion(XContentLocation location, String field, Object expectedValue) {
super(location, field, expectedValue);
}
@Override

View File

@ -21,6 +21,7 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
@ -36,8 +37,8 @@ public class LessThanOrEqualToAssertion extends Assertion {
private static final ESLogger logger = Loggers.getLogger(LessThanOrEqualToAssertion.class);
public LessThanOrEqualToAssertion(String field, Object expectedValue) {
super(field, expectedValue);
public LessThanOrEqualToAssertion(XContentLocation location, String field, Object expectedValue) {
super(location, field, expectedValue);
}
@Override

View File

@ -21,6 +21,7 @@ package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.XContentLocation;
import java.util.List;
import java.util.Locale;
@ -45,8 +46,8 @@ public class MatchAssertion extends Assertion {
private static final ESLogger logger = Loggers.getLogger(MatchAssertion.class);
public MatchAssertion(String field, Object expectedValue) {
super(field, expectedValue);
public MatchAssertion(XContentLocation location, String field, Object expectedValue) {
super(location, field, expectedValue);
}
@Override

View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.test.rest.yaml.section;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import java.io.IOException;
@ -32,7 +33,12 @@ import java.util.Map;
*/
public class SetSection implements ExecutableSection {
private Map<String, String> stash = new HashMap<>();
private final Map<String, String> stash = new HashMap<>();
private final XContentLocation location;
public SetSection(XContentLocation location) {
this.location = location;
}
public void addSet(String responseField, String stashedField) {
stash.put(responseField, stashedField);
@ -42,6 +48,11 @@ public class SetSection implements ExecutableSection {
return stash;
}
@Override
public XContentLocation getLocation() {
return location;
}
@Override
public void execute(ClientYamlTestExecutionContext executionContext) throws IOException {
for (Map.Entry<String, String> entry : stash.entrySet()) {