fix #35262 define deprecations of API's as a whole and urls (#39063)

* fix #35262 define deprecations of API's as a whole and urls

* document hot threads deprecated paths

* deprecate scroll_id as part of the URL, documented only as part of the body which is a safer behaviour as well

* use version numbers up to patch version

* rest spec parser picks up deprecated paths as paths too

(cherry picked from commit 7e06023e7603b7584bfd9ee4e8a1ccd82c208ce7)

rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json

@@ -3,8 +3,15 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-scroll.html",
     "methods": ["DELETE"],
     "url": {
+      "path": "/_search/scroll",
+      "paths": [ "/_search/scroll"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
           "path" : "/_search/scroll/{scroll_id}",
-      "paths": ["/_search/scroll/{scroll_id}", "/_search/scroll"],
+          "description" : "A scroll id can be quite large and should be specified as part of the body"
+        }
+      ],
       "parts": {
         "scroll_id": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/count.json

@@ -4,7 +4,14 @@
     "methods": ["POST", "GET"],
     "url": {
       "path": "/_count",
-      "paths": ["/_count", "/{index}/_count", "/{index}/{type}/_count"],
+      "paths": ["/_count", "/{index}/_count"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_count",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/create.json

@@ -4,7 +4,14 @@
     "methods": ["PUT","POST"],
     "url": {
       "path": "/{index}/_create/{id}",
-      "paths": ["/{index}/_create/{id}", "/{index}/{type}/{id}/_create"],
+      "paths": ["/{index}/_create/{id}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}/_create",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/delete.json

@@ -4,7 +4,14 @@
     "methods": ["DELETE"],
     "url": {
       "path": "/{index}/_doc/{id}",
-      "paths": ["/{index}/{type}/{id}", "/{index}/_doc/{id}"],
+      "paths": ["/{index}/_doc/{id}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json

@@ -4,7 +4,14 @@
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_delete_by_query",
-      "paths": ["/{index}/_delete_by_query", "/{index}/{type}/_delete_by_query"],
+      "paths": ["/{index}/_delete_by_query"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_delete_by_query",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "comment": "most things below this are just copied from search.json",
       "parts": {
         "index": {

rest-api-spec/src/main/resources/rest-api-spec/api/exists.json

@@ -4,7 +4,14 @@
     "methods": ["HEAD"],
     "url": {
       "path": "/{index}/_doc/{id}",
-      "paths": ["/{index}/_doc/{id}", "/{index}/{type}/{id}"],
+      "paths": ["/{index}/_doc/{id}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json

@@ -4,7 +4,14 @@
     "methods": ["HEAD"],
     "url": {
       "path": "/{index}/_source/{id}",
-      "paths": ["/{index}/_source/{id}", "/{index}/{type}/{id}/_source"],
+      "paths": ["/{index}/_source/{id}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}/_source",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/explain.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/{index}/_explain/{id}",
-      "paths": ["/{index}/_explain/{id}", "/{index}/{type}/{id}/_explain"],
+      "paths": ["/{index}/_explain/{id}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}/_explain",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/get.json

@@ -4,7 +4,14 @@
     "methods": ["GET"],
     "url": {
       "path": "/{index}/_doc/{id}",
-      "paths": ["/{index}/_doc/{id}", "/{index}/{type}/{id}"],
+      "paths": ["/{index}/_doc/{id}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json

@@ -4,7 +4,14 @@
     "methods": ["GET"],
     "url": {
       "path": "/{index}/_source/{id}",
-      "paths": ["/{index}/_source/{id}", "/{index}/{type}/{id}/_source"],
+      "paths": ["/{index}/_source/{id}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}/_source",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/index.json

@@ -4,7 +4,19 @@
     "methods": ["POST", "PUT"],
     "url": {
       "path": "/{index}/_doc",
-      "paths": ["/{index}/{type}", "/{index}/{type}/{id}", "/{index}/_doc/{id}", "/{index}/_doc"],
+      "paths": ["/{index}/_doc/{id}", "/{index}/_doc"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}",
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json

@@ -2,6 +2,10 @@
   "indices.exists_type": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-types-exists.html",
     "methods": ["HEAD"],
+    "deprecated" : {
+      "version" : "7.0.0",
+      "description" : "Types are being removed from elasticsearch and therefor this API is on the way out. Read more here: https://www.elastic.co/guide/en/elasticsearch/reference/master/removal-of-types.html"
+    },
     "url": {
       "path": "/{index}/_mapping/{type}",
       "paths": ["/{index}/_mapping/{type}"],

rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json

@@ -4,7 +4,19 @@
     "methods": ["GET"],
     "url": {
       "path": "/_mapping/field/{fields}",
-      "paths": ["/_mapping/field/{fields}", "/{index}/_mapping/field/{fields}", "/_mapping/{type}/field/{fields}", "/{index}/_mapping/{type}/field/{fields}"],
+      "paths": ["/_mapping/field/{fields}", "/{index}/_mapping/field/{fields}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/_mapping/{type}/field/{fields}",
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/_mapping/{type}/field/{fields}",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json

@@ -4,7 +4,19 @@
     "methods": ["GET"],
     "url": {
       "path": "/_mapping",
-      "paths": ["/_mapping", "/{index}/_mapping", "/_mapping/{type}", "/{index}/_mapping/{type}"],
+      "paths": ["/_mapping", "/{index}/_mapping"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/_mapping/{type}",
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/_mapping/{type}",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json

@@ -3,8 +3,45 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-put-mapping.html",
     "methods": ["PUT", "POST"],
     "url": {
+      "path": "{index}/_mapping",
+      "paths": ["{index}/_mapping"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
           "path" : "/{index}/{type}/_mapping",
-      "paths": ["/{index}/{type}/_mapping", "/{index}/_mapping/{type}", "/_mapping/{type}", "/{index}/{type}/_mappings", "/{index}/_mappings/{type}", "/_mappings/{type}", "{index}/_mappings", "{index}/_mapping"],
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/_mapping/{type}",
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_mappings",
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/_mappings/{type}",
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/_mappings/{type}",
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "{index}/_mappings",
+          "description" : "The plural mappings is accepted but only /_mapping is documented"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/_mapping/{type}",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_validate/query",
-      "paths": ["/_validate/query", "/{index}/_validate/query", "/{index}/{type}/_validate/query"],
+      "paths": ["/_validate/query", "/{index}/_validate/query"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_validate/query",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/mget.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_mget",
-      "paths": ["/_mget", "/{index}/_mget", "/{index}/{type}/_mget"],
+      "paths": ["/_mget", "/{index}/_mget"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_mget",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_msearch",
-      "paths": ["/_msearch", "/{index}/_msearch", "/{index}/{type}/_msearch"],
+      "paths": ["/_msearch", "/{index}/_msearch"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_msearch",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
          "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_msearch/template",
-      "paths": ["/_msearch/template", "/{index}/_msearch/template", "/{index}/{type}/_msearch/template"],
+      "paths": ["/_msearch/template", "/{index}/_msearch/template"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_msearch/template",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json

@@ -4,7 +4,14 @@
     "methods" : ["GET", "POST"],
     "url" : {
       "path" : "/_mtermvectors",
-      "paths" : ["/_mtermvectors", "/{index}/_mtermvectors", "/{index}/{type}/_mtermvectors"],
+      "paths" : ["/_mtermvectors", "/{index}/_mtermvectors"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_mtermvectors",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts" : {
         "index" : {
          "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json

@@ -4,7 +4,39 @@
     "methods": ["GET"],
     "url": {
       "path": "/_nodes/hot_threads",
-      "paths": ["/_cluster/nodes/hotthreads", "/_cluster/nodes/hot_threads", "/_cluster/nodes/{node_id}/hotthreads", "/_cluster/nodes/{node_id}/hot_threads", "/_nodes/hotthreads", "/_nodes/hot_threads", "/_nodes/{node_id}/hotthreads", "/_nodes/{node_id}/hot_threads"],
+      "paths": ["/_nodes/hot_threads",  "/_nodes/{node_id}/hot_threads"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/_cluster/nodes/hotthreads",
+          "description" : "The hot threads API accepts `hotthreads` but only `hot_threads` is documented"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/_cluster/nodes/{node_id}/hotthreads",
+          "description" : "The hot threads API accepts `hotthreads` but only `hot_threads` is documented"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/_nodes/hotthreads",
+          "description" : "The hot threads API accepts `hotthreads` but only `hot_threads` is documented"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/_nodes/{node_id}/hotthreads",
+          "description" : "The hot threads API accepts `hotthreads` but only `hot_threads` is documented"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/_cluster/nodes/hot_threads",
+          "description" : "The hot accepts /_cluster/nodes as prefix for backwards compatibility reasons"
+        },
+        {
+          "version" : "7.0.0",
+          "path" :"/_cluster/nodes/{node_id}/hot_threads",
+          "description" : "The hot accepts /_cluster/nodes as prefix for backwards compatibility reasons"
+        }
+      ],
       "parts": {
         "node_id": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_search/scroll",
-      "paths": ["/_search/scroll", "/_search/scroll/{scroll_id}"],
+      "paths": ["/_search/scroll"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/_search/scroll/{scroll_id}",
+          "description" : "A scroll id can be quite large and should be specified as part of the body"
+        }
+      ],
       "parts": {
         "scroll_id": {
           "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/search.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_search",
-      "paths": ["/_search", "/{index}/_search", "/{index}/{type}/_search"],
+      "paths": ["/_search", "/{index}/_search"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_search",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
          "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_search/template",
-      "paths": ["/_search/template", "/{index}/_search/template", "/{index}/{type}/_search/template"],
+      "paths": ["/_search/template", "/{index}/_search/template"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_search/template",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type" : "list",

rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json

@@ -4,8 +4,19 @@
     "methods" : ["GET", "POST"],
     "url" : {
       "path" : "/{index}/_termvectors/{id}",
-      "paths" : ["/{index}/_termvectors/{id}", "/{index}/_termvectors/",
+      "paths" : ["/{index}/_termvectors/{id}", "/{index}/_termvectors"],
-                 "/{index}/{type}/{id}/_termvectors", "/{index}/{type}/_termvectors"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}/_termvectors",
+          "description" : "Specifying types in urls has been deprecated"
+        },
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_termvectors",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts" : {
         "index" : {
          "type" : "string",

rest-api-spec/src/main/resources/rest-api-spec/api/update.json

@@ -4,7 +4,14 @@
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_update/{id}",
-      "paths": ["/{index}/_update/{id}", "/{index}/{type}/{id}/_update"],
+      "paths": ["/{index}/_update/{id}"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/{id}/_update",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "id": {
           "type": "string",

rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json

@@ -4,7 +4,14 @@
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_update_by_query",
-      "paths": ["/{index}/_update_by_query", "/{index}/{type}/_update_by_query"],
+      "paths": ["/{index}/_update_by_query"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_update_by_query",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "comment": "most things below this are just copied from search.json",
       "parts": {
         "index": {

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java

@@ -70,11 +70,16 @@ public class ClientYamlSuiteRestApiParser {
                         }
                         if (parser.currentToken() == XContentParser.Token.START_ARRAY && "paths".equals(currentFieldName)) {
                             while (parser.nextToken() == XContentParser.Token.VALUE_STRING) {
-                                String path = parser.text();
+                                addPathToApi(parser.text(), restApi);
-                                if (restApi.getPaths().contains(path)) {
+                            }
-                                    throw new IllegalArgumentException("Found duplicate path [" + path + "]");
+                        }
+                        if (parser.currentToken() == XContentParser.Token.START_ARRAY && "deprecated_paths".equals(currentFieldName)) {
+                            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                                if (parser.currentToken() == XContentParser.Token.FIELD_NAME && "path".equals(parser.currentName()))
+                                {
+                                    parser.nextToken();
+                                    addPathToApi(parser.text(), restApi);
                                 }
-                                restApi.addPath(path);
                             }
                         }
 
@@ -142,6 +147,13 @@ public class ClientYamlSuiteRestApiParser {
         return restApi;
     }
 
+    private void addPathToApi(String path, ClientYamlSuiteRestApi restApi) {
+        if (restApi.getPaths().contains(path)) {
+            throw new IllegalArgumentException("Found duplicate path [" + path + "]");
+        }
+        restApi.addPath(path);
+    }
+
     private static class Parameter {
         private boolean required;
         public boolean isRequired() {

x-pack/plugin/src/test/resources/rest-api-spec/api/graph.explore.json

@@ -4,7 +4,14 @@
     "methods": ["GET", "POST"],
     "url": {
       "path": "/{index}/_graph/explore",
-      "paths": ["/{index}/_graph/explore", "/{index}/{type}/_graph/explore"],
+      "paths": ["/{index}/_graph/explore"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/{index}/{type}/_graph/explore",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts" : {
         "index": {
          "type" : "list",

x-pack/plugin/src/test/resources/rest-api-spec/api/monitoring.bulk.json

@@ -4,7 +4,14 @@
     "methods": ["POST", "PUT"],
     "url": {
       "path": "/_monitoring/bulk",
-      "paths": ["/_monitoring/bulk", "/_monitoring/{type}/bulk"],
+      "paths": ["/_monitoring/bulk"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/_monitoring/{type}/bulk",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "type": {
           "type" : "string",

x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup_search.json

@@ -4,7 +4,14 @@
     "methods": [ "GET", "POST" ],
     "url": {
       "path": "/{index}/_rollup_search",
-      "paths": [ "{index}/_rollup_search", "{index}/{type}/_rollup_search" ],
+      "paths": [ "{index}/_rollup_search"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "{index}/{type}/_rollup_search",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ],
       "parts": {
         "index": {
           "type": "list",