Merge pull request #19091 from javanna/test/nuke_response_body_assertion

Consolidate docs snippets testing in our REST test infra

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -131,7 +131,8 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         }
 
         private void response(Snippet response) {
-            current.println("  - response_body: |")
+            current.println("  - match: ")
+            current.println("      \$body: ")
             response.contents.eachLine { current.println("        $it") }
         }
 

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -1263,26 +1263,8 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]MockEngineSupport.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchAssertions.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]json[/\\]JsonPath.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanEqualToParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]LessThanOrEqualToParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]LessThanParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]RestTestSuiteParseContext.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]RestTestSuiteParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]GreaterThanAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]GreaterThanEqualToAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]LengthAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]LessThanAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]LessThanOrEqualToAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]MatchAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]spec[/\\]RestApiParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]support[/\\]FileUtils.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSDirectoryService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSIndexStore.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]FileUtilsTests.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]JsonPathTests.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]RestTestParserTests.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]test[/\\]InternalTestClusterTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliTool.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]indices[/\\]settings[/\\]RestGetSettingsAction.java" checks="LineLength" />

docs/reference/ingest/ingest-node.asciidoc

@@ -46,7 +46,6 @@ PUT _ingest/pipeline/my-pipeline-id
         "value": "bar"
       }
     }
-    // other processors
   ]
 }
 --------------------------------------------------
@@ -83,7 +82,6 @@ Example response:
             "value": "bar"
           }
         }
-        // other processors
       ]
     }
   } ]

rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml

@@ -17,12 +17,8 @@
       headers:
         Accept: application/yaml
 
-  - match:
+  - match: {0.alias: test_alias}
-      $body: |
+  - match: {0.index: test}
-        /^---\n
+  - match: {0.filter: "-"}
-          -\s+alias:\s+"test_alias"\s+
+  - match: {0.routing\.index: "-"}
-              index:\s+"test"\s+
+  - match: {0.routing\.search: "-"}
-              filter:\s+"-"\s+
-              routing.index:\s+"-"\s+
-              routing.search:\s+"-"\s+$/
-

rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml

@@ -1,8 +1,5 @@
 ---
-"Simple alias with yaml body through format argument":
+"Simple alias with json body through format argument":
-
-  - skip:
-      features: yaml
 
   - do:
         indices.create:
@@ -15,15 +12,10 @@
 
   - do:
       cat.aliases:
-        format: yaml
+        format: json
-
-  - match:
-      $body: |
-        /^---\n
-          -\s+alias:\s+"test_alias"\s+
-              index:\s+"test"\s+
-              filter:\s+"-"\s+
-              routing.index:\s+"-"\s+
-              routing.search:\s+"-"\s+$/
-
 
+  - match: {0.alias: test_alias}
+  - match: {0.index: test}
+  - match: {0.filter: "-"}
+  - match: {0.routing\.index: "-"}
+  - match: {0.routing\.search: "-"}

rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml

@@ -18,13 +18,9 @@
         type: _all
         id:   1
 
-  - match:
+  - match: {_index: "test_1"}
-      $body: |
+  - match: {_type: "test"}
-        /^---\n
+  - match: {_id: "1"}
-          _index:\s+\"test_1"\n
+  - match: {_version: 1}
-          _type:\s+"test"\n
+  - match: {found: true}
-          _id:\s+"1"\n
+  - match: { _source: { body: foo }}
-          _version:\s+1\n
-          found:\s+true\n
-          _source:\n
-          \s+body:\s+"foo"\n$/

test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java

@@ -16,11 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.json;
+package org.elasticsearch.test.rest;
 
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.Stash;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -28,22 +27,23 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * Holds a json object and allows to extract specific values from it
+ * Holds an object and allows to extract specific values from it given their path
  */
-public class JsonPath {
+public class ObjectPath {
 
-    final String json;
+    private final Object object;
-    final Map<String, Object> jsonMap;
 
-    public JsonPath(String json) throws IOException {
+    public static ObjectPath createFromXContent(XContent xContent, String input) throws IOException {
-        this.json = json;
+        try (XContentParser parser = xContent.createParser(input)) {
-        this.jsonMap = convertToMap(json);
+            if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
+                return new ObjectPath(parser.listOrderedMap());
+            }
+            return new ObjectPath(parser.mapOrdered());
+        }
     }
 
-    private static Map<String, Object> convertToMap(String json) throws IOException {
+    public ObjectPath(Object object) {
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(json)) {
+        this.object = object;
-            return parser.mapOrdered();
-        }
     }
 
     /**
@@ -58,7 +58,7 @@ public class JsonPath {
      */
     public Object evaluate(String path, Stash stash) throws IOException {
         String[] parts = parsePath(path);
-        Object object = jsonMap;
+        Object object = this.object;
         for (String part : parts) {
             object = evaluate(part, object, stash);
             if (object == null) {
@@ -71,7 +71,7 @@ public class JsonPath {
     @SuppressWarnings("unchecked")
     private Object evaluate(String key, Object object, Stash stash) throws IOException {
         if (stash.isStashedValue(key)) {
-            key = stash.unstashValue(key).toString();
+            key = stash.getValue(key).toString();
         }
 
         if (object instanceof Map) {
@@ -84,7 +84,8 @@ public class JsonPath {
             } catch (NumberFormatException e) {
                 throw new IllegalArgumentException("element was a list, but [" + key + "] was not numeric", e);
             } catch (IndexOutOfBoundsException e) {
-                throw new IllegalArgumentException("element was a list with " + list.size() + " elements, but [" + key + "] was out of bounds", e);
+                throw new IllegalArgumentException("element was a list with " + list.size() +
+                        " elements, but [" + key + "] was out of bounds", e);
             }
         }
 

test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java

@@ -67,7 +67,7 @@ public class RestTestExecutionContext implements Closeable {
         HashMap<String, String> requestParams = new HashMap<>(params);
         for (Map.Entry<String, String> entry : requestParams.entrySet()) {
             if (stash.isStashedValue(entry.getValue())) {
-                entry.setValue(stash.unstashValue(entry.getValue()).toString());
+                entry.setValue(stash.getValue(entry.getValue()).toString());
             }
         }
 
@@ -76,7 +76,7 @@ public class RestTestExecutionContext implements Closeable {
         try {
             response = callApiInternal(apiName, requestParams, body, headers);
             //we always stash the last response body
-            stash.stashResponse(response);
+            stash.stashValue("body", response.getBody());
             return response;
         } catch(ResponseException e) {
             response = new RestTestResponse(e);
@@ -90,12 +90,12 @@ public class RestTestExecutionContext implements Closeable {
         }
 
         if (bodies.size() == 1) {
-            return bodyAsString(stash.unstashMap(bodies.get(0)));
+            return bodyAsString(stash.replaceStashedValues(bodies.get(0)));
         }
 
         StringBuilder bodyBuilder = new StringBuilder();
         for (Map<String, Object> body : bodies) {
-            bodyBuilder.append(bodyAsString(stash.unstashMap(body))).append("\n");
+            bodyBuilder.append(bodyAsString(stash.replaceStashedValues(body))).append("\n");
         }
         return bodyBuilder.toString();
     }

test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java

@@ -24,7 +24,6 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.test.rest.client.RestTestResponse;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -42,7 +41,7 @@ public class Stash implements ToXContent {
     public static final Stash EMPTY = new Stash();
 
     private final Map<String, Object> stash = new HashMap<>();
-    private RestTestResponse response;
+    private final ObjectPath stashObjectPath = new ObjectPath(stash);
 
     /**
      * Allows to saved a specific field in the stash as key-value pair
@@ -55,12 +54,6 @@ public class Stash implements ToXContent {
         }
     }
 
-    public void stashResponse(RestTestResponse response) throws IOException {
-        // TODO we can almost certainly save time by lazily evaluating the body
-        stashValue("body", response.getBody());
-        this.response = response;
-    }
-
     /**
      * Clears the previously stashed values
      */
@@ -69,7 +62,8 @@ public class Stash implements ToXContent {
     }
 
     /**
-     * Tells whether a particular value needs to be looked up in the stash
+     * Tells whether a particular key needs to be looked up in the stash based on its name.
+     * Returns true if the string representation of the key starts with "$", false otherwise
      * The stash contains fields eventually extracted from previous responses that can be reused
      * as arguments for following requests (e.g. scroll_id)
      */
@@ -82,28 +76,23 @@ public class Stash implements ToXContent {
     }
 
     /**
-     * Extracts a value from the current stash
+     * Retrieves a value from the current stash.
      * The stash contains fields eventually extracted from previous responses that can be reused
      * as arguments for following requests (e.g. scroll_id)
      */
-    public Object unstashValue(String value) throws IOException {
+    public Object getValue(String key) throws IOException {
-        if (value.startsWith("$body.")) {
+        Object stashedValue = stashObjectPath.evaluate(key.substring(1));
-            if (response == null) {
-                return null;
-            }
-            return response.evaluate(value.substring("$body".length()), this);
-        }
-        Object stashedValue = stash.get(value.substring(1));
         if (stashedValue == null) {
-            throw new IllegalArgumentException("stashed value not found for key [" + value + "]");
+            throw new IllegalArgumentException("stashed value not found for key [" + key + "]");
         }
         return stashedValue;
     }
 
     /**
-     * Recursively unstashes map values if needed
+     * Goes recursively against each map entry and replaces any string value starting with "$" with its
+     * corresponding value retrieved from the stash
      */
-    public Map<String, Object> unstashMap(Map<String, Object> map) throws IOException {
+    public Map<String, Object> replaceStashedValues(Map<String, Object> map) throws IOException {
         Map<String, Object> copy = new HashMap<>(map);
         unstashObject(copy);
         return copy;
@@ -116,7 +105,7 @@ public class Stash implements ToXContent {
             for (int i = 0; i < list.size(); i++) {
                 Object o = list.get(i);
                 if (isStashedValue(o)) {
-                    list.set(i, unstashValue(o.toString()));
+                    list.set(i, getValue(o.toString()));
                 } else {
                     unstashObject(o);
                 }
@@ -126,7 +115,7 @@ public class Stash implements ToXContent {
             Map<String, Object> map = (Map) obj;
             for (Map.Entry<String, Object> entry : map.entrySet()) {
                 if (isStashedValue(entry.getValue())) {
-                    entry.setValue(unstashValue(entry.getValue().toString()));
+                    entry.setValue(getValue(entry.getValue().toString()));
                 } else {
                     unstashObject(entry.getValue());
                 }

test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java

@@ -23,23 +23,24 @@ import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.test.rest.Stash;
-import org.elasticsearch.test.rest.json.JsonPath;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 
 /**
  * Response obtained from a REST call, eagerly reads the response body into a string for later optional parsing.
- * Supports parsing the response body as json when needed and returning specific values extracted from it.
+ * Supports parsing the response body when needed and returning specific values extracted from it.
  */
 public class RestTestResponse {
 
     private final Response response;
     private final String body;
-    private JsonPath parsedResponse;
+    private ObjectPath parsedResponse;
 
-    public RestTestResponse(Response response) {
+    public RestTestResponse(Response response) throws IOException {
         this.response = response;
         if (response.getEntity() != null) {
             try {
@@ -53,11 +54,24 @@ public class RestTestResponse {
         } else {
             this.body = null;
         }
+        parseResponseBody();
     }
 
-    public RestTestResponse(ResponseException responseException) {
+    public RestTestResponse(ResponseException responseException) throws IOException {
         this.response = responseException.getResponse();
         this.body = responseException.getResponseBody();
+        parseResponseBody();
+    }
+
+    private void parseResponseBody() throws IOException {
+        if (body != null) {
+            String contentType = response.getHeader("Content-Type");
+            XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType);
+            //skip parsing if we got text back (e.g. if we called _cat apis)
+            if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
+                this.parsedResponse = ObjectPath.createFromXContent(xContentType.xContent(), body);
+            }
+        }
     }
 
     public int getStatusCode() {
@@ -73,11 +87,7 @@ public class RestTestResponse {
      * Might be a string or a json object parsed as a map.
      */
     public Object getBody() throws IOException {
-        if (isJson()) {
+        if (parsedResponse != null) {
-            JsonPath parsedResponse = parsedResponse();
-            if (parsedResponse == null) {
-                return null;
-            }
             return parsedResponse.evaluate("");
         }
         return body;
@@ -95,23 +105,21 @@ public class RestTestResponse {
     }
 
     /**
-     * Parses the response body as json and extracts a specific value from it (identified by the provided path)
+     * Parses the response body and extracts a specific value from it (identified by the provided path)
      */
     public Object evaluate(String path) throws IOException {
         return evaluate(path, Stash.EMPTY);
     }
 
     /**
-     * Parses the response body as json and extracts a specific value from it (identified by the provided path)
+     * Parses the response body and extracts a specific value from it (identified by the provided path)
      */
     public Object evaluate(String path, Stash stash) throws IOException {
         if (response == null) {
             return null;
         }
 
-        JsonPath jsonPath = parsedResponse();
+        if (parsedResponse == null) {
-
-        if (jsonPath == null) {
             //special case: api that don't support body (e.g. exists) return true if 200, false if 404, even if no body
             //is_true: '' means the response had no body but the client returned true (caused by 200)
             //is_false: '' means the response had no body but the client returned false (caused by 404)
@@ -121,21 +129,6 @@ public class RestTestResponse {
             return null;
         }
 
-        return jsonPath.evaluate(path, stash);
+        return parsedResponse.evaluate(path, stash);
-    }
-
-    private boolean isJson() {
-        String contentType = response.getHeader("Content-Type");
-        return contentType != null && contentType.contains("application/json");
-    }
-
-    private JsonPath parsedResponse() throws IOException {
-        if (parsedResponse != null) {
-            return parsedResponse;
-        }
-        if (response == null || body == null) {
-            return null;
-        }
-        return parsedResponse = new JsonPath(body);
     }
 }

test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java

@@ -33,7 +33,8 @@ public class GreaterThanEqualToParser implements RestTestFragmentParser<GreaterT
     public GreaterThanEqualToAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
         Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
         if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new RestTestParseException("gte section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName());
+            throw new RestTestParseException("gte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
         }
         return new GreaterThanEqualToAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
     }

test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java

@@ -32,7 +32,8 @@ public class GreaterThanParser implements RestTestFragmentParser<GreaterThanAsse
     public GreaterThanAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
         Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
         if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new RestTestParseException("gt section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName());
+            throw new RestTestParseException("gt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
         }
         return new GreaterThanAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
     }

test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java

@@ -33,7 +33,8 @@ public class LessThanOrEqualToParser implements RestTestFragmentParser<LessThanO
     public LessThanOrEqualToAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
         Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
         if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new RestTestParseException("lte section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName());
+            throw new RestTestParseException("lte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
         }
         return new LessThanOrEqualToAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
     }

test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java

@@ -32,7 +32,8 @@ public class LessThanParser implements RestTestFragmentParser<LessThanAssertion>
     public LessThanAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
         Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
         if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new RestTestParseException("lt section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName());
+            throw new RestTestParseException("lt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
         }
         return new LessThanAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
     }

test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java

@@ -18,22 +18,21 @@
  */
 package org.elasticsearch.test.rest.parser;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.test.rest.section.DoSection;
 import org.elasticsearch.test.rest.section.ExecutableSection;
-import org.elasticsearch.test.rest.section.ResponseBodyAssertion;
 import org.elasticsearch.test.rest.section.SetupSection;
 import org.elasticsearch.test.rest.section.SkipSection;
 import org.elasticsearch.test.rest.section.TeardownSection;
 import org.elasticsearch.test.rest.section.TestSection;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * Context shared across the whole tests parse phase.
  * Provides shared parse methods and holds information needed to parse the test sections (e.g. es version)
@@ -57,7 +56,6 @@ public class RestTestSuiteParseContext {
         EXECUTABLE_SECTIONS_PARSERS.put("lt", new LessThanParser());
         EXECUTABLE_SECTIONS_PARSERS.put("lte", new LessThanOrEqualToParser());
         EXECUTABLE_SECTIONS_PARSERS.put("length", new LengthParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("response_body", ResponseBodyAssertion.PARSER);
     }
 
     private final String api;
@@ -160,7 +158,8 @@ public class RestTestSuiteParseContext {
             token = parser.nextToken();
         }
         if (token != XContentParser.Token.FIELD_NAME) {
-            throw new RestTestParseException("malformed test section: field name expected but found " + token + " at " + parser.getTokenLocation());
+            throw new RestTestParseException("malformed test section: field name expected but found " + token + " at "
+                    + parser.getTokenLocation());
         }
     }
 

test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java

@@ -71,7 +71,8 @@ public class RestTestSuiteParser implements RestTestFragmentParser<RestTestSuite
         XContentParser parser = parseContext.parser();
 
         parser.nextToken();
-        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was " + parser.currentToken();
+        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
+                + parser.currentToken();
 
         RestTestSuite restTestSuite = new RestTestSuite(parseContext.getApi(), parseContext.getSuiteName());
 
@@ -90,7 +91,8 @@ public class RestTestSuiteParser implements RestTestFragmentParser<RestTestSuite
 
             TestSection testSection = parseContext.parseTestSection();
             if (!restTestSuite.addTestSection(testSection)) {
-                throw new RestTestParseException("duplicate test section [" + testSection.getName() + "] found in [" + restTestSuite.getPath() + "]");
+                throw new RestTestParseException("duplicate test section [" + testSection.getName() + "] found in ["
+                        + restTestSuite.getPath() + "]");
             }
         }
 

test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java

@@ -48,18 +48,18 @@ public abstract class Assertion implements ExecutableSection {
         if (expectedValue instanceof Map) {
             @SuppressWarnings("unchecked")
             Map<String, Object> map = (Map<String, Object>) expectedValue;
-            return executionContext.stash().unstashMap(map);
+            return executionContext.stash().replaceStashedValues(map);
         }
 
         if (executionContext.stash().isStashedValue(expectedValue)) {
-            return executionContext.stash().unstashValue(expectedValue.toString());
+            return executionContext.stash().getValue(expectedValue.toString());
         }
         return expectedValue;
     }
 
     protected final Object getActualValue(RestTestExecutionContext executionContext) throws IOException {
         if (executionContext.stash().isStashedValue(field)) {
-            return executionContext.stash().unstashValue(field);
+            return executionContext.stash().getValue(field);
         }
         return executionContext.response(field);
     }

test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java

@@ -43,8 +43,10 @@ public class GreaterThanAssertion extends Assertion {
     @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is greater than [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class));
+        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
-        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class));
+                actualValue, instanceOf(Comparable.class));
+        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])",
+                expectedValue, instanceOf(Comparable.class));
         try {
             assertThat(errorMessage(), (Comparable) actualValue, greaterThan((Comparable) expectedValue));
         } catch (ClassCastException e) {

test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java

@@ -43,8 +43,10 @@ public class GreaterThanEqualToAssertion extends Assertion {
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is greater than or equal to [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class));
+        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
-        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class));
+                actualValue, instanceOf(Comparable.class));
+        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])",
+                expectedValue, instanceOf(Comparable.class));
         try {
             assertThat(errorMessage(), (Comparable) actualValue, greaterThanOrEqualTo((Comparable) expectedValue));
         } catch (ClassCastException e) {

test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java

@@ -44,7 +44,8 @@ public class LengthAssertion extends Assertion {
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] has length [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("expected value of [" + getField() + "] is not numeric (got [" + expectedValue.getClass() + "]", expectedValue, instanceOf(Number.class));
+        assertThat("expected value of [" + getField() + "] is not numeric (got [" + expectedValue.getClass() + "]",
+                expectedValue, instanceOf(Number.class));
         int length = ((Number) expectedValue).intValue();
         if (actualValue instanceof String) {
             assertThat(errorMessage(), ((String) actualValue).length(), equalTo(length));

test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java

@@ -44,8 +44,10 @@ public class LessThanAssertion extends Assertion {
     @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is less than [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class));
+        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
-        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class));
+                actualValue, instanceOf(Comparable.class));
+        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])",
+                expectedValue, instanceOf(Comparable.class));
         try {
             assertThat(errorMessage(), (Comparable) actualValue, lessThan((Comparable) expectedValue));
         } catch (ClassCastException e) {

test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java

@@ -43,8 +43,10 @@ public class LessThanOrEqualToAssertion  extends Assertion {
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is less than or equal to [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class));
+        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
-        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class));
+                actualValue, instanceOf(Comparable.class));
+        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])",
+                expectedValue, instanceOf(Comparable.class));
         try {
             assertThat(errorMessage(), (Comparable) actualValue, lessThanOrEqualTo((Comparable) expectedValue));
         } catch (ClassCastException e) {

test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java

@@ -18,15 +18,21 @@
  */
 package org.elasticsearch.test.rest.section;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 
 /**
@@ -45,12 +51,12 @@ public class MatchAssertion extends Assertion {
 
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
-
         //if the value is wrapped into / it is a regexp (e.g. /s+d+/)
         if (expectedValue instanceof String) {
             String expValue = ((String) expectedValue).trim();
             if (expValue.length() > 2 && expValue.startsWith("/") && expValue.endsWith("/")) {
-                assertThat("field [" + getField() + "] was expected to be of type String but is an instanceof [" + safeClass(actualValue) + "]", actualValue, instanceOf(String.class));
+                assertThat("field [" + getField() + "] was expected to be of type String but is an instanceof [" +
+                        safeClass(actualValue) + "]", actualValue, instanceOf(String.class));
                 String stringValue = (String) actualValue;
                 String regex = expValue.substring(1, expValue.length() - 1);
                 logger.trace("assert that [{}] matches [{}]", stringValue, regex);
@@ -60,20 +66,131 @@ public class MatchAssertion extends Assertion {
             }
         }
 
-        assertThat(errorMessage(), actualValue, notNullValue());
+        assertNotNull("field [" + getField() + "] is null", actualValue);
         logger.trace("assert that [{}] matches [{}] (field [{}])", actualValue, expectedValue, getField());
-        if (!actualValue.getClass().equals(safeClass(expectedValue))) {
+        if (actualValue.getClass().equals(safeClass(expectedValue)) == false) {
             if (actualValue instanceof Number && expectedValue instanceof Number) {
                 //Double 1.0 is equal to Integer 1
-                assertThat(errorMessage(), ((Number) actualValue).doubleValue(), equalTo(((Number) expectedValue).doubleValue()));
+                assertThat("field [" + getField() + "] doesn't match the expected value",
+                        ((Number) actualValue).doubleValue(), equalTo(((Number) expectedValue).doubleValue()));
                 return;
             }
         }
 
-        assertThat(errorMessage(), actualValue, equalTo(expectedValue));
+        if (expectedValue.equals(actualValue) == false) {
+            FailureMessage message = new FailureMessage(getField());
+            message.compare(getField(), actualValue, expectedValue);
+            throw new AssertionError(message.message);
+        }
     }
 
-    private String errorMessage() {
+    private static class FailureMessage {
-        return "field [" + getField() + "] doesn't match the expected value";
+        private final StringBuilder message;
+        private int indent = 0;
+
+        private FailureMessage(String field) {
+            this.message = new StringBuilder(field + " didn't match the expected value:\n");
+        }
+
+        private void compareMaps(Map<String, Object> actual, Map<String, Object> expected) {
+            actual = new TreeMap<>(actual);
+            expected = new TreeMap<>(expected);
+            for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
+                compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
+            }
+            for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
+                field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
+            }
+        }
+
+        private void compareLists(List<Object> actual, List<Object> expected) {
+            int i = 0;
+            while (i < actual.size() && i < expected.size()) {
+                compare(Integer.toString(i), actual.get(i), expected.get(i));
+                i++;
+            }
+            if (actual.size() == expected.size()) {
+                return;
+            }
+            indent();
+            if (actual.size() < expected.size()) {
+                message.append("expected [").append(expected.size() - i).append("] more entries\n");
+                return;
+            }
+            message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
+        }
+
+        private void compare(String field, @Nullable Object actual, Object expected) {
+            if (expected instanceof Map) {
+                if (actual == null) {
+                    field(field, "expected map but not found");
+                    return;
+                }
+                if (false == actual instanceof Map) {
+                    field(field, "expected map but found [" + actual + "]");
+                    return;
+                }
+                @SuppressWarnings("unchecked")
+                Map<String, Object> expectedMap = (Map<String, Object>) expected;
+                @SuppressWarnings("unchecked")
+                Map<String, Object> actualMap = (Map<String, Object>) actual;
+                if (expectedMap.isEmpty() && actualMap.isEmpty()) {
+                    field(field, "same [empty map]");
+                    return;
+                }
+                field(field, null);
+                indent += 1;
+                compareMaps(actualMap, expectedMap);
+                indent -= 1;
+                return;
+            }
+            if (expected instanceof List) {
+                if (actual == null) {
+                    field(field, "expected list but not found");
+                    return;
+                }
+                if (false == actual instanceof List) {
+                    field(field, "expected list but found [" + actual + "]");
+                    return;
+                }
+                @SuppressWarnings("unchecked")
+                List<Object> expectedList = (List<Object>) expected;
+                @SuppressWarnings("unchecked")
+                List<Object> actualList = (List<Object>) actual;
+                if (expectedList.isEmpty() && actualList.isEmpty()) {
+                    field(field, "same [empty list]");
+                    return;
+                }
+                field(field, null);
+                indent += 1;
+                compareLists(actualList, expectedList);
+                indent -= 1;
+                return;
+            }
+            if (actual == null) {
+                field(field, "expected [" + expected + "] but not found");
+                return;
+            }
+            if (Objects.equals(expected, actual)) {
+                field(field, "same [" + expected + "]");
+                return;
+            }
+            field(field, "expected [" + expected + "] but was [" + actual + "]");
+        }
+
+        private void indent() {
+            for (int i = 0; i < indent; i++) {
+                message.append("  ");
+            }
+        }
+
+        private void field(Object name, String info) {
+            indent();
+            message.append(String.format(Locale.ROOT, "%30s: ", name));
+            if (info != null) {
+                message.append(info);
+            }
+            message.append('\n');
+        }
     }
 }

test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java

@@ -1,170 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.section;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TreeMap;
-
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.parser.RestTestFragmentParser;
-import org.elasticsearch.test.rest.parser.RestTestParseException;
-import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext;
-
-/**
- * Checks that the response body matches some text. 
- */
-public class ResponseBodyAssertion extends Assertion {
-    public static final RestTestFragmentParser<ResponseBodyAssertion> PARSER = new RestTestFragmentParser<ResponseBodyAssertion>() {
-        @Override
-        public ResponseBodyAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
-            try (XContentParser parser = JsonXContent.jsonXContent.createParser(parseContext.parseField())) {
-                return new ResponseBodyAssertion("$body", parser.map());
-            }
-        }
-    };
-
-    private ResponseBodyAssertion(String field, Map<String, Object> expectedValue) {
-        super(field, expectedValue);
-    }
-
-    @Override
-    protected void doAssert(Object actualValue, Object expectedValue) {
-        if (false == expectedValue.equals(actualValue)) {
-            @SuppressWarnings("unchecked")
-            Map<String, Object> actual = (Map<String, Object>) actualValue;
-            @SuppressWarnings("unchecked")
-            Map<String, Object> expected = (Map<String, Object>) expectedValue;
-            FailureMessage message = new FailureMessage();
-            message.compareMaps(actual, expected);
-            throw new AssertionError(message.message);
-        }
-    }
-
-    private class FailureMessage {
-        private final StringBuilder message = new StringBuilder("body didn't match the expected value:\n");
-        private int indent = 0;
-
-        private void compareMaps(Map<String, Object> actual, Map<String, Object> expected) {
-            actual = new TreeMap<>(actual);
-            expected = new TreeMap<>(expected);
-            for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
-                compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
-            }
-            for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
-                field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
-            }
-        }
-
-        private void compareLists(List<Object> actual, List<Object> expected) {
-            int i = 0;
-            while (i < actual.size() && i < expected.size()) {
-                compare(i, actual.get(i), expected.get(i));
-                i++;
-            }
-            if (actual.size() == expected.size()) {
-                return;
-            }
-            indent();
-            if (actual.size() < expected.size()) {
-                message.append("expected [").append(expected.size() - i).append("] more entries\n");
-                return;
-            }
-            message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
-        }
-
-        private void compare(Object field, @Nullable Object actual, Object expected) {
-            if (expected instanceof Map) {
-                if (actual == null) {
-                    field(field, "expected map but not found");
-                    return;
-                }
-                if (false == actual instanceof Map) {
-                    field(field, "expected map but found [" + actual + "]");
-                    return;
-                }
-                @SuppressWarnings("unchecked")
-                Map<String, Object> expectedMap = (Map<String, Object>) expected;
-                @SuppressWarnings("unchecked")
-                Map<String, Object> actualMap = (Map<String, Object>) actual;
-                if (expectedMap.isEmpty() && actualMap.isEmpty()) {
-                    field(field, "same [empty map]");
-                    return;
-                }
-                field(field, null);
-                indent += 1;
-                compareMaps(actualMap, expectedMap);
-                indent -= 1;
-                return;
-            }
-            if (expected instanceof List) {
-                if (actual == null) {
-                    field(field, "expected list but not found");
-                    return;
-                }
-                if (false == actual instanceof List) {
-                    field(field, "expected list but found [" + actual + "]");
-                    return;
-                }
-                @SuppressWarnings("unchecked")
-                List<Object> expectedList = (List<Object>) expected;
-                @SuppressWarnings("unchecked")
-                List<Object> actualList = (List<Object>) actual;
-                if (expectedList.isEmpty() && actualList.isEmpty()) {
-                    field(field, "same [empty list]");
-                    return;
-                }
-                field(field, null);
-                indent += 1;
-                compareLists(actualList, expectedList);
-                indent -= 1;
-                return;
-            }
-            if (actual == null) {
-                field(field, "expected [" + expected + "] but not found");
-                return;
-            }
-            if (Objects.equals(expected, actual)) {
-                field(field, "same [" + expected + "]");
-                return;
-            }
-            field(field, "expected [" + expected + "] but was [" + actual + "]");
-        }
-
-        private void indent() {
-            for (int i = 0; i < indent; i++) {
-                message.append("  ");
-            }
-        }
-
-        private void field(Object name, String info) {
-            indent();
-            message.append(String.format(Locale.ROOT, "%30s: ", name));
-            if (info != null) {
-                message.append(info);
-            }
-            message.append('\n');
-        }
-    }
-}

test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java

@@ -34,7 +34,7 @@ import java.util.List;
  */
 public final class Features {
 
-    private static final List<String> SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "yaml");
+    private static final List<String> SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers");
 
     private Features() {
 

test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java

@@ -78,7 +78,8 @@ public final class FileUtils {
      * Each input path can either be a single file (the .yaml suffix is optional) or a directory.
      * Each path is looked up in the classpath, or optionally from {@code fileSystem} if its not null.
      */
-    public static Map<String, Set<Path>> findYamlSuites(FileSystem fileSystem, String optionalPathPrefix, final String... paths) throws IOException {
+    public static Map<String, Set<Path>> findYamlSuites(FileSystem fileSystem, String optionalPathPrefix, final String... paths)
+            throws IOException {
         Map<String, Set<Path>> yamlSuites = new HashMap<>();
         for (String path : paths) {
             collectFiles(resolveFile(fileSystem, optionalPathPrefix, path, YAML_SUFFIX), YAML_SUFFIX, yamlSuites);
@@ -86,7 +87,8 @@ public final class FileUtils {
         return yamlSuites;
     }
 
-    private static Path resolveFile(FileSystem fileSystem, String optionalPathPrefix, String path, String optionalFileSuffix) throws IOException {
+    private static Path resolveFile(FileSystem fileSystem, String optionalPathPrefix, String path, String optionalFileSuffix)
+            throws IOException {
         if (fileSystem != null) {
             Path file = findFile(fileSystem, path, optionalFileSuffix);
             if (!lenientExists(file)) {
@@ -94,7 +96,8 @@ public final class FileUtils {
                 String newPath = optionalPathPrefix + "/" + path;
                 file = findFile(fileSystem, newPath, optionalFileSuffix);
                 if (!lenientExists(file)) {
-                    throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: " + optionalFileSuffix);
+                    throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: "
+                            + optionalFileSuffix);
                 }
             }
             return file;

test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java

@@ -84,7 +84,8 @@ public class FileUtilsTests extends ESTestCase {
         assertSingleFile(yamlSuites.get(dir.getFileName().toString()), dir.getFileName().toString(), file.getFileName().toString());
 
         //load from external file (optional extension)
-        yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test", dir.resolve("test_loading").toAbsolutePath().toString());
+        yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test",
+                dir.resolve("test_loading").toAbsolutePath().toString());
         assertThat(yamlSuites, notNullValue());
         assertThat(yamlSuites.size(), equalTo(1));
         assertThat(yamlSuites.containsKey(dir.getFileName().toString()), equalTo(true));

test/framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java

@@ -1,159 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.test;
-
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.Stash;
-import org.elasticsearch.test.rest.json.JsonPath;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class JsonPathTests extends ESTestCase {
-    public void testEvaluateObjectPathEscape() throws Exception {
-        String json = "{ \"field1\": { \"field2.field3\" : \"value2\" } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2\\.field3");
-        assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateObjectPathWithDoubleDot() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1..field2");
-        assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateObjectPathEndsWithDot() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2.");
-        assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateString() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2");
-        assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateInteger() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : 333 } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2");
-        assertThat(object, instanceOf(Integer.class));
-        assertThat((Integer)object, equalTo(333));
-    }
-
-    public void testEvaluateDouble() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : 3.55 } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2");
-        assertThat(object, instanceOf(Double.class));
-        assertThat((Double)object, equalTo(3.55));
-    }
-
-    public void testEvaluateArray() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.array1");
-        assertThat(object, instanceOf(List.class));
-        List list = (List) object;
-        assertThat(list.size(), equalTo(2));
-        assertThat(list.get(0), instanceOf(String.class));
-        assertThat((String)list.get(0), equalTo("value1"));
-        assertThat(list.get(1), instanceOf(String.class));
-        assertThat((String)list.get(1), equalTo("value2"));
-    }
-
-    public void testEvaluateArrayElement() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.array1.1");
-        assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateArrayElementObject() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.array1.1.element");
-        assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateArrayElementObjectWrongPath() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.array2.1.element");
-        assertThat(object, nullValue());
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testEvaluateObjectKeys() throws Exception {
-        String json = "{ \"metadata\": { \"templates\" : {\"template_1\": { \"field\" : \"value\"}, \"template_2\": { \"field\" : \"value\"} } } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("metadata.templates");
-        assertThat(object, instanceOf(Map.class));
-        Map<String, Object> map = (Map<String, Object>)object;
-        assertThat(map.size(), equalTo(2));
-        Set<String> strings = map.keySet();
-        assertThat(strings, contains("template_1", "template_2"));
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testEvaluateEmptyPath() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("");
-        assertThat(object, notNullValue());
-        assertThat(object, instanceOf(Map.class));
-        assertThat(((Map<String, Object>)object).containsKey("field1"), equalTo(true));
-    }
-
-    public void testEvaluateStashInPropertyName() throws Exception {
-        String json = "{ \"field1\": { \"elements\" : {\"element1\": \"value1\"}}}";
-        JsonPath jsonPath = new JsonPath(json);
-        try {
-            jsonPath.evaluate("field1.$placeholder.element1");
-            fail("evaluate should have failed due to unresolved placeholder");
-        } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("stashed value not found for key [$placeholder]"));
-        }
-
-        Stash stash = new Stash();
-        stash.stashValue("placeholder", "elements");
-        Object object = jsonPath.evaluate("field1.$placeholder.element1", stash);
-        assertThat(object, notNullValue());
-        assertThat(object.toString(), equalTo("value1"));
-    }
-}

test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java

@@ -0,0 +1,237 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.rest.test;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.elasticsearch.test.rest.Stash;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ObjectPathTests extends ESTestCase {
+
+    private static XContentBuilder randomXContentBuilder() throws IOException {
+        //only string based formats are supported, no cbor nor smile
+        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.YAML);
+        return XContentBuilder.builder(XContentFactory.xContent(xContentType));
+    }
+
+    public void testEvaluateObjectPathEscape() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.field("field2.field3", "value2");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        Object object = objectPath.evaluate("field1.field2\\.field3");
+        assertThat(object, instanceOf(String.class));
+        assertThat(object, equalTo("value2"));
+    }
+
+    public void testEvaluateObjectPathWithDots() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.field("field2", "value2");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        Object object = objectPath.evaluate("field1..field2");
+        assertThat(object, instanceOf(String.class));
+        assertThat(object, equalTo("value2"));
+        object = objectPath.evaluate("field1.field2.");
+        assertThat(object, instanceOf(String.class));
+        assertThat(object, equalTo("value2"));
+        object = objectPath.evaluate("field1.field2");
+        assertThat(object, instanceOf(String.class));
+        assertThat(object, equalTo("value2"));
+    }
+
+    public void testEvaluateInteger() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.field("field2", 333);
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        Object object = objectPath.evaluate("field1.field2");
+        assertThat(object, instanceOf(Integer.class));
+        assertThat(object, equalTo(333));
+    }
+
+    public void testEvaluateDouble() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.field("field2", 3.55);
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        Object object = objectPath.evaluate("field1.field2");
+        assertThat(object, instanceOf(Double.class));
+        assertThat(object, equalTo(3.55));
+    }
+
+    public void testEvaluateArray() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.array("array1", "value1", "value2");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        Object object = objectPath.evaluate("field1.array1");
+        assertThat(object, instanceOf(List.class));
+        List list = (List) object;
+        assertThat(list.size(), equalTo(2));
+        assertThat(list.get(0), instanceOf(String.class));
+        assertThat(list.get(0), equalTo("value1"));
+        assertThat(list.get(1), instanceOf(String.class));
+        assertThat(list.get(1), equalTo("value2"));
+        object = objectPath.evaluate("field1.array1.1");
+        assertThat(object, instanceOf(String.class));
+        assertThat(object, equalTo("value2"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testEvaluateArrayElementObject() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.startArray("array1");
+        xContentBuilder.startObject();
+        xContentBuilder.field("element", "value1");
+        xContentBuilder.endObject();
+        xContentBuilder.startObject();
+        xContentBuilder.field("element", "value2");
+        xContentBuilder.endObject();
+        xContentBuilder.endArray();
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        Object object = objectPath.evaluate("field1.array1.1.element");
+        assertThat(object, instanceOf(String.class));
+        assertThat(object, equalTo("value2"));
+        object = objectPath.evaluate("");
+        assertThat(object, notNullValue());
+        assertThat(object, instanceOf(Map.class));
+        assertThat(((Map<String, Object>)object).containsKey("field1"), equalTo(true));
+        object = objectPath.evaluate("field1.array2.1.element");
+        assertThat(object, nullValue());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testEvaluateObjectKeys() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("metadata");
+        xContentBuilder.startObject("templates");
+        xContentBuilder.startObject("template_1");
+        xContentBuilder.field("field", "value");
+        xContentBuilder.endObject();
+        xContentBuilder.startObject("template_2");
+        xContentBuilder.field("field", "value");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        Object object = objectPath.evaluate("metadata.templates");
+        assertThat(object, instanceOf(Map.class));
+        Map<String, Object> map = (Map<String, Object>)object;
+        assertThat(map.size(), equalTo(2));
+        Set<String> strings = map.keySet();
+        assertThat(strings, contains("template_1", "template_2"));
+    }
+
+    public void testEvaluateStashInPropertyName() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.startObject("elements");
+        xContentBuilder.field("element1", "value1");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
+        try {
+            objectPath.evaluate("field1.$placeholder.element1");
+            fail("evaluate should have failed due to unresolved placeholder");
+        } catch(IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("stashed value not found for key [$placeholder]"));
+        }
+
+        Stash stash = new Stash();
+        stash.stashValue("placeholder", "elements");
+        Object object = objectPath.evaluate("field1.$placeholder.element1", stash);
+        assertThat(object, notNullValue());
+        assertThat(object.toString(), equalTo("value1"));
+
+        Map<String, Object> stashedObject = new HashMap<>();
+        stashedObject.put("subobject", "elements");
+        stash.stashValue("object", stashedObject);
+        object = objectPath.evaluate("field1.$object\\.subobject.element1", stash);
+        assertThat(object, notNullValue());
+        assertThat(object.toString(), equalTo("value1"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testEvaluateArrayAsRoot() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startArray();
+        xContentBuilder.startObject();
+        xContentBuilder.field("alias", "test_alias1");
+        xContentBuilder.field("index", "test1");
+        xContentBuilder.endObject();
+        xContentBuilder.startObject();
+        xContentBuilder.field("alias", "test_alias2");
+        xContentBuilder.field("index", "test2");
+        xContentBuilder.endObject();
+        xContentBuilder.endArray();
+        ObjectPath objectPath = ObjectPath.createFromXContent(XContentFactory.xContent(XContentType.YAML), xContentBuilder.string());
+        Object object = objectPath.evaluate("");
+        assertThat(object, notNullValue());
+        assertThat(object, instanceOf(List.class));
+        assertThat(((List<Object>)object).size(), equalTo(2));
+        object = objectPath.evaluate("0");
+        assertThat(object, notNullValue());
+        assertThat(object, instanceOf(Map.class));
+        assertThat(((Map<String, Object>)object).get("alias"), equalTo("test_alias1"));
+        object = objectPath.evaluate("1.index");
+        assertThat(object, notNullValue());
+        assertThat(object, instanceOf(String.class));
+        assertThat(object, equalTo("test2"));
+    }
+}

test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java

@@ -115,7 +115,8 @@ public class RestTestParserTests extends ESTestCase {
             assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1));
             assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create"));
             assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
-            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), equalTo("test_index"));
+            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
+                    equalTo("test_index"));
         } else {
             assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
         }
@@ -154,7 +155,8 @@ public class RestTestParserTests extends ESTestCase {
 
         assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Get type mapping - pre 1.0"));
         assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(false));
-        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getReason(), equalTo("for newer versions the index name is always returned"));
+        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getReason(),
+                equalTo("for newer versions the index name is always returned"));
         assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
         assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getUpperVersion(), equalTo(Version.CURRENT));
         assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(3));