Deprecate REST access to System Indices (#63274) (Original #60945)

This PR adds deprecation warnings when accessing System Indices via the REST layer. At this time, these warnings are only enabled for Snapshot builds by default, to allow projects external to Elasticsearch additional time to adjust their access patterns.

Deprecation warnings will be triggered by all REST requests which access registered System Indices, except for purpose-specific APIs which access System Indices as an implementation detail a few specific APIs which will continue to allow access to system indices by default:

- `GET _cluster/health`
- `GET {index}/_recovery`
- `GET _cluster/allocation/explain`
- `GET _cluster/state`
- `POST _cluster/reroute`
- `GET {index}/_stats`
- `GET {index}/_segments`
- `GET {index}/_shard_stores`
- `GET _cat/[indices,aliases,health,recovery,shards,segments]`

Deprecation warnings for accessing system indices take the form:
```
this request accesses system indices: [.some_system_index], but in a future major version, direct access to system indices will be prevented by default
```

docs/reference/api-conventions.asciidoc

@@ -42,6 +42,14 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
 
 The defaults settings for the above parameters depend on the API being used.
 
+Some indices (hereafter "system indices") are used by various system
+modules and/or plugins to store state or configuration. These indices
+are not intended to be accessed directly, and accessing them directly is
+deprecated. In the next major version, access to these indices will no longer be
+allowed to prevent accidental operations that may cause problems with
+Elasticsearch features which depend on the consistency of data in these
+indices.
+
 Some multi-target APIs that can target indices also support the following query
 string parameter:
 

modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java

@@ -139,6 +139,11 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
             return "kibana_" + super.getName();
         }
 
+        @Override
+        public boolean allowSystemIndexAccessByDefault() {
+            return true;
+        }
+
         @Override
         public List<Route> routes() {
             return Collections.unmodifiableList(

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java

@@ -45,6 +45,8 @@ public class RestMultiSearchTemplateActionTests extends RestActionTestCase {
             .withPath("/some_index/some_type/_msearch/template")
             .withContent(bytesContent, XContentType.JSON)
             .build();
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
 
         dispatchRequest(request);
         assertWarnings(RestMultiSearchTemplateAction.TYPES_DEPRECATION_MESSAGE);
@@ -59,6 +61,8 @@ public class RestMultiSearchTemplateActionTests extends RestActionTestCase {
             .withPath("/some_index/_msearch/template")
             .withContent(bytesContent, XContentType.JSON)
             .build();
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
 
         dispatchRequest(request);
         assertWarnings(RestMultiSearchTemplateAction.TYPES_DEPRECATION_MESSAGE);

modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java

@@ -36,6 +36,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndices;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -61,7 +62,8 @@ public class ReindexSourceTargetValidationTests extends ESTestCase {
                 .put(index("baz"), true)
                 .put(index("source", "source_multi"), true)
                 .put(index("source2", "source_multi"), true)).build();
-    private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver();
+    private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER =
+        new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
     private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER,
             new SystemIndices(new HashMap<>()));

modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestDeleteByQueryActionTests.java

@@ -24,8 +24,8 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.test.rest.RestActionTestCase;
-
 import org.junit.Before;
+
 import java.io.IOException;
 
 import static java.util.Collections.emptyList;
@@ -44,6 +44,10 @@ public class RestDeleteByQueryActionTests extends RestActionTestCase {
             .withMethod(RestRequest.Method.POST)
             .withPath("/some_index/some_type/_delete_by_query")
             .build();
+
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteLocallyVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
 
         // checks the type in the URL is propagated correctly to the request object

modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java

@@ -31,8 +31,8 @@ import org.elasticsearch.test.rest.RestActionTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Arrays;
+import java.util.Collections;
 
 import static java.util.Collections.singletonMap;
 
@@ -102,6 +102,10 @@ public class RestReindexActionTests extends RestActionTestCase {
         }
         b.endObject();
         requestBuilder.withContent(new BytesArray(BytesReference.bytes(b).toBytesRef()), XContentType.JSON);
+
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteLocallyVerifier((arg1, arg2) -> null);
+
         dispatchRequest(requestBuilder.build());
         assertWarnings(ReindexRequest.TYPES_DEPRECATION_MESSAGE);
     }
@@ -123,6 +127,10 @@ public class RestReindexActionTests extends RestActionTestCase {
         }
         b.endObject();
         requestBuilder.withContent(new BytesArray(BytesReference.bytes(b).toBytesRef()), XContentType.JSON);
+
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteLocallyVerifier((arg1, arg2) -> null);
+
         dispatchRequest(requestBuilder.build());
         assertWarnings(ReindexRequest.TYPES_DEPRECATION_MESSAGE);
     }

modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestUpdateByQueryActionTests.java

@@ -24,8 +24,8 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.test.rest.RestActionTestCase;
-
 import org.junit.Before;
+
 import java.io.IOException;
 
 import static java.util.Collections.emptyList;
@@ -45,6 +45,10 @@ public class RestUpdateByQueryActionTests extends RestActionTestCase {
             .withMethod(RestRequest.Method.POST)
             .withPath("/some_index/some_type/_update_by_query")
             .build();
+
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteLocallyVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
 
         // checks the type in the URL is propagated correctly to the request object

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml

@@ -74,6 +74,8 @@
 
 ---
 "Rethrottle to -1 which turns off throttling":
+  - skip:
+      features: warnings
   # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
   # and a small batch size on the request
   - do:
@@ -95,6 +97,7 @@
         index:   test
         body:    { "text": "test" }
   - do:
+
       indices.refresh: {}
 
   - do:
@@ -121,6 +124,8 @@
         task_id: $task
 
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
 
   - do:

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml

@@ -62,6 +62,8 @@
 
 ---
 "Multiple slices with wait_for_completion=false":
+  - skip:
+      features: warnings
   - do:
       index:
         index:   test
@@ -151,8 +153,12 @@
 
   # Only the "parent" reindex task wrote its status to the tasks index though
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       search:
         rest_total_hits_as_int: true
         index: .tasks
@@ -165,6 +171,8 @@
 
 ---
 "Multiple slices with rethrottle":
+  - skip:
+      features: warnings
   - do:
       index:
         index:   test
@@ -196,7 +204,8 @@
         id:      6
         body:    { "text": "test" }
   - do:
-      indices.refresh: {}
+      indices.refresh:
+        index: test
 
   # Start the task with a requests_per_second that should make it take a very long time
   - do:
@@ -259,8 +268,12 @@
 
   # Only the "parent" reindex task wrote its status to the tasks index though
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       search:
         rest_total_hits_as_int: true
         index: .tasks

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml

@@ -58,6 +58,8 @@
 
 ---
 "Multiple slices with wait_for_completion=false":
+  - skip:
+      features: warnings
   - do:
       index:
         index:   source
@@ -160,8 +162,12 @@
 
   # Only the "parent" reindex task wrote its status to the tasks index though
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       search:
         rest_total_hits_as_int: true
         index: .tasks
@@ -170,6 +176,8 @@
 
 ---
 "Multiple slices with rethrottle":
+  - skip:
+      features: warnings
   - do:
       index:
         index:   source
@@ -272,8 +280,12 @@
 
   # Only the "parent" reindex task wrote its status to the tasks index though
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       search:
         rest_total_hits_as_int: true
         index: .tasks

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml

@@ -54,6 +54,8 @@
 
 ---
 "Multiple slices with wait_for_completion=false":
+  - skip:
+      features: warnings
   - do:
       index:
         index:   test
@@ -143,8 +145,12 @@
 
   # Only the "parent" reindex task wrote its status to the tasks index though
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       search:
         rest_total_hits_as_int: true
         index: .tasks
@@ -152,6 +158,8 @@
 
 ---
 "Multiple slices with rethrottle":
+  - skip:
+      features: warnings
   - do:
       index:
         index:   test
@@ -246,8 +254,12 @@
 
   # Only the "parent" reindex task wrote its status to the tasks index though
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       indices.refresh: {}
   - do:
+      warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       search:
         rest_total_hits_as_int: true
         index: .tasks

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -42,6 +42,7 @@ import org.elasticsearch.rest.action.document.RestIndexAction;
 import org.elasticsearch.rest.action.document.RestUpdateAction;
 import org.elasticsearch.rest.action.search.RestExplainAction;
 import org.elasticsearch.test.NotEqualMessageBuilder;
+import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
 import org.junit.Before;
@@ -62,6 +63,7 @@ import java.util.regex.Pattern;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ENFORCEMENT_VERSION;
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -69,6 +71,7 @@ import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -342,7 +345,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}");
             client().performRequest(shrinkIndexRequest);
 
-            client().performRequest(new Request("POST", "/_refresh"));
+            refreshAllIndices();
         } else {
             numDocs = countOfIndexedRandomDocuments();
         }
@@ -427,7 +430,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             numDocs = countOfIndexedRandomDocuments();
         }
 
-        client().performRequest(new Request("POST", "/_refresh"));
+        refreshAllIndices();
 
         Map<?, ?> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
         assertNoFailures(response);
@@ -1457,73 +1460,108 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
     }
 
-    public void testCreateSystemIndexInOldVersion() throws Exception {
-        assumeTrue("only run on old cluster", isRunningAgainstOldCluster());
-        // create index
-        Request createTestIndex = new Request("PUT", "/test_index_old");
-        createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0, \"index.number_of_shards\": 1}}");
-        client().performRequest(createTestIndex);
+    @SuppressWarnings("unchecked")
+    public void testSystemIndexMetadataIsUpgraded() throws Exception {
+        final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " +
+            "access to system indices will be prevented by default";
+        if (isRunningAgainstOldCluster()) {
+            // create index
+            Request createTestIndex = new Request("PUT", "/test_index_old");
+            createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0, \"index.number_of_shards\": 1}}");
+            client().performRequest(createTestIndex);
 
-        Request bulk = new Request("POST", "/_bulk");
-        bulk.addParameter("refresh", "true");
-        bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\", \"_type\" : \"_doc\"}}\n" +
-            "{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
+            Request bulk = new Request("POST", "/_bulk");
+            bulk.addParameter("refresh", "true");
+            bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\", \"_type\" : \"_doc\"}}\n" +
+                "{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
         if (isRunningAgainstAncientCluster() == false) {
             bulk.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE));
         }
-        client().performRequest(bulk);
+            client().performRequest(bulk);
 
-        // start a async reindex job
-        Request reindex = new Request("POST", "/_reindex");
-        reindex.setJsonEntity(
-            "{\n" +
-                "  \"source\":{\n" +
-                "    \"index\":\"test_index_old\"\n" +
-                "  },\n" +
-                "  \"dest\":{\n" +
-                "    \"index\":\"test_index_reindex\"\n" +
-                "  }\n" +
-                "}");
-        reindex.addParameter("wait_for_completion", "false");
-        Map<String, Object> response = entityAsMap(client().performRequest(reindex));
-        String taskId = (String) response.get("task");
+            // start a async reindex job
+            Request reindex = new Request("POST", "/_reindex");
+            reindex.setJsonEntity(
+                "{\n" +
+                    "  \"source\":{\n" +
+                    "    \"index\":\"test_index_old\"\n" +
+                    "  },\n" +
+                    "  \"dest\":{\n" +
+                    "    \"index\":\"test_index_reindex\"\n" +
+                    "  }\n" +
+                    "}");
+            reindex.addParameter("wait_for_completion", "false");
+            Map<String, Object> response = entityAsMap(client().performRequest(reindex));
+            String taskId = (String) response.get("task");
 
-        // wait for task
-        Request getTask = new Request("GET", "/_tasks/" + taskId);
-        getTask.addParameter("wait_for_completion", "true");
-        client().performRequest(getTask);
+            // wait for task
+            Request getTask = new Request("GET", "/_tasks/" + taskId);
+            getTask.addParameter("wait_for_completion", "true");
+            client().performRequest(getTask);
 
-        // make sure .tasks index exists
-        assertBusy(() -> {
+            // make sure .tasks index exists
             Request getTasksIndex = new Request("GET", "/.tasks");
+            getTasksIndex.addParameter("allow_no_indices", "false");
             if (getOldClusterVersion().onOrAfter(Version.V_6_7_0) && getOldClusterVersion().before(Version.V_7_0_0)) {
                 getTasksIndex.addParameter("include_type_name", "false");
             }
-            assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
-        });
-    }
 
-    @SuppressWarnings("unchecked" +
-        "")
-    public void testSystemIndexGetsUpdatedMetadata() throws Exception {
-        assumeFalse("only run in upgraded cluster", isRunningAgainstOldCluster());
+            getTasksIndex.setOptions(expectVersionSpecificWarnings(v -> {
+                v.current(systemIndexWarning);
+                v.compatible(systemIndexWarning);
+            }));
+            assertBusy(() -> {
+                try {
+                    assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
+                } catch (ResponseException e) {
+                    throw new AssertionError(".tasks index does not exist yet");
+                }
+            });
 
-        assertBusy(() -> {
-            Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
-            Map<String, Object> response = entityAsMap(client().performRequest(clusterStateRequest));
-            Map<String, Object> metadata = (Map<String, Object>) response.get("metadata");
-            assertNotNull(metadata);
-            Map<String, Object> indices = (Map<String, Object>) metadata.get("indices");
-            assertNotNull(indices);
+            // If we are on 7.x create an alias that includes both a system index and a non-system index so we can be sure it gets
+            // upgraded properly. If we're already on 8.x, skip this part of the test.
+            if (minimumNodeVersion().before(SYSTEM_INDEX_ENFORCEMENT_VERSION)) {
+                // Create an alias to make sure it gets upgraded properly
+                Request putAliasRequest = new Request("POST", "/_aliases");
+                putAliasRequest.setJsonEntity("{\n" +
+                    "  \"actions\": [\n" +
+                    "    {\"add\":  {\"index\":  \".tasks\", \"alias\": \"test-system-alias\"}},\n" +
+                    "    {\"add\":  {\"index\":  \"test_index_reindex\", \"alias\": \"test-system-alias\"}}\n" +
+                    "  ]\n" +
+                    "}");
+                assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200));
+            }
+        } else {
+            assertBusy(() -> {
+                Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
+                Map<String, Object> indices = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest)))
+                    .get("metadata.indices");
 
-            Map<String, Object> tasksIndex = (Map<String, Object>) indices.get(".tasks");
-            assertNotNull(tasksIndex);
-            assertThat(tasksIndex.get("system"), is(true));
+                // Make sure our non-system index is still non-system
+                assertThat(new XContentTestUtils.JsonMapView(indices).get("test_index_old.system"), is(false));
 
-            Map<String, Object> testIndex = (Map<String, Object>) indices.get("test_index_old");
-            assertNotNull(testIndex);
-            assertThat(testIndex.get("system"), is(false));
-        });
+                // Can't get the .tasks index via JsonMapView because it splits on `.`
+                assertThat(indices, hasKey(".tasks"));
+                XContentTestUtils.JsonMapView tasksIndex = new XContentTestUtils.JsonMapView((Map<String, Object>) indices.get(".tasks"));
+                assertThat(tasksIndex.get("system"), is(true));
+
+                // If .tasks was created in a 7.x version, it should have an alias on it that we need to make sure got upgraded properly.
+                final String tasksCreatedVersionString = tasksIndex.get("settings.index.version.created");
+                assertThat(tasksCreatedVersionString, notNullValue());
+                final Version tasksCreatedVersion = Version.fromId(Integer.parseInt(tasksCreatedVersionString));
+                if (tasksCreatedVersion.before(SYSTEM_INDEX_ENFORCEMENT_VERSION)) {
+                    // Verify that the alias survived the upgrade
+                    Request getAliasRequest = new Request("GET", "/_alias/test-system-alias");
+                    getAliasRequest.setOptions(expectVersionSpecificWarnings(v -> {
+                        v.current(systemIndexWarning);
+                        v.compatible(systemIndexWarning);
+                    }));
+                    Map<String, Object> aliasResponse = entityAsMap(client().performRequest(getAliasRequest));
+                    assertThat(aliasResponse, hasKey(".tasks"));
+                    assertThat(aliasResponse, hasKey("test_index_reindex"));
+                }
+            });
+        }
     }
 
     public void testEnableSoftDeletesOnRestore() throws Exception {

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java

@@ -21,84 +21,121 @@ package org.elasticsearch.upgrades;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.XContentTestUtils.JsonMapView;
 
 import java.util.Map;
 
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ENFORCEMENT_VERSION;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class SystemIndicesUpgradeIT extends AbstractRollingTestCase {
 
-    public void testOldDoesntHaveSystemIndexMetadata() throws Exception {
-        assumeTrue("only run in old cluster", CLUSTER_TYPE == ClusterType.OLD);
-        // create index
-        Request createTestIndex = new Request("PUT", "/test_index_old");
-        createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0, \"index.number_of_shards\": 1}}");
-        client().performRequest(createTestIndex);
+    @SuppressWarnings("unchecked")
+    public void testSystemIndicesUpgrades() throws Exception {
+        final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " +
+            "access to system indices will be prevented by default";
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            // create index
+            Request createTestIndex = new Request("PUT", "/test_index_old");
+            createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}}");
+            client().performRequest(createTestIndex);
 
-        Request bulk = new Request("POST", "/_bulk");
-        bulk.addParameter("refresh", "true");
-        if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) {
-            bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\", \"_type\" : \"_doc\"}}\n" +
-                "{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
-        } else {
-            bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\"}\n" +
-                "{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
-        }
-        client().performRequest(bulk);
+            Request bulk = new Request("POST", "/_bulk");
+            bulk.addParameter("refresh", "true");
+            if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) {
+                bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\", \"_type\" : \"_doc\"}}\n" +
+                    "{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
+            } else {
+                bulk.setJsonEntity("{\"index\": {\"_index\": \"test_index_old\"}\n" +
+                    "{\"f1\": \"v1\", \"f2\": \"v2\"}\n");
+            }
+            client().performRequest(bulk);
 
-        // start a async reindex job
-        Request reindex = new Request("POST", "/_reindex");
-        reindex.setJsonEntity(
-            "{\n" +
-                "  \"source\":{\n" +
-                "    \"index\":\"test_index_old\"\n" +
-                "  },\n" +
-                "  \"dest\":{\n" +
-                "    \"index\":\"test_index_reindex\"\n" +
-                "  }\n" +
-                "}");
-        reindex.addParameter("wait_for_completion", "false");
-        Map<String, Object> response = entityAsMap(client().performRequest(reindex));
-        String taskId = (String) response.get("task");
+            // start a async reindex job
+            Request reindex = new Request("POST", "/_reindex");
+            reindex.setJsonEntity(
+                "{\n" +
+                    "  \"source\":{\n" +
+                    "    \"index\":\"test_index_old\"\n" +
+                    "  },\n" +
+                    "  \"dest\":{\n" +
+                    "    \"index\":\"test_index_reindex\"\n" +
+                    "  }\n" +
+                    "}");
+            reindex.addParameter("wait_for_completion", "false");
+            Map<String, Object> response = entityAsMap(client().performRequest(reindex));
+            String taskId = (String) response.get("task");
 
-        // wait for task
-        Request getTask = new Request("GET", "/_tasks/" + taskId);
-        getTask.addParameter("wait_for_completion", "true");
-        client().performRequest(getTask);
+            // wait for task
+            Request getTask = new Request("GET", "/_tasks/" + taskId);
+            getTask.addParameter("wait_for_completion", "true");
+            client().performRequest(getTask);
 
-        // make sure .tasks index exists
-        assertBusy(() -> {
+            // make sure .tasks index exists
             Request getTasksIndex = new Request("GET", "/.tasks");
+            getTasksIndex.addParameter("allow_no_indices", "false");
             if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) {
                 getTasksIndex.addParameter("include_type_name", "false");
             }
-            assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
-        });
-    }
 
-    public void testMixedCluster() {
-        assumeTrue("nothing to do in mixed cluster", CLUSTER_TYPE == ClusterType.MIXED);
-    }
+            getTasksIndex.setOptions(expectVersionSpecificWarnings(v -> {
+                v.current(systemIndexWarning);
+                v.compatible(systemIndexWarning);
+            }));
+            assertBusy(() -> {
+                try {
+                    assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
+                } catch (ResponseException e) {
+                    throw new AssertionError(".tasks index does not exist yet");
+                }
+            });
 
-    @SuppressWarnings("unchecked")
-    public void testUpgradedCluster() throws Exception {
-        assumeTrue("only run on upgraded cluster", CLUSTER_TYPE == ClusterType.UPGRADED);
+            // If we are on 7.x create an alias that includes both a system index and a non-system index so we can be sure it gets
+            // upgraded properly. If we're already on 8.x, skip this part of the test.
+            if (minimumNodeVersion().before(SYSTEM_INDEX_ENFORCEMENT_VERSION)) {
+                // Create an alias to make sure it gets upgraded properly
+                Request putAliasRequest = new Request("POST", "/_aliases");
+                putAliasRequest.setJsonEntity("{\n" +
+                    "  \"actions\": [\n" +
+                    "    {\"add\":  {\"index\":  \".tasks\", \"alias\": \"test-system-alias\"}},\n" +
+                    "    {\"add\":  {\"index\":  \"test_index_reindex\", \"alias\": \"test-system-alias\"}}\n" +
+                    "  ]\n" +
+                    "}");
+                assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200));
+            }
+        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+            assertBusy(() -> {
+                Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
+                Map<String, Object> indices = new JsonMapView(entityAsMap(client().performRequest(clusterStateRequest)))
+                    .get("metadata.indices");
 
-        assertBusy(() -> {
-            Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
-            Map<String, Object> response = entityAsMap(client().performRequest(clusterStateRequest));
-            Map<String, Object> metadata = (Map<String, Object>) response.get("metadata");
-            assertNotNull(metadata);
-            Map<String, Object> indices = (Map<String, Object>) metadata.get("indices");
-            assertNotNull(indices);
+                // Make sure our non-system index is still non-system
+                assertThat(new JsonMapView(indices).get("test_index_old.system"), is(false));
 
-            Map<String, Object> tasksIndex = (Map<String, Object>) indices.get(".tasks");
-            assertNotNull(tasksIndex);
-            assertThat(tasksIndex.get("system"), is(true));
+                // Can't get the .tasks index via JsonMapView because it splits on `.`
+                assertThat(indices, hasKey(".tasks"));
+                JsonMapView tasksIndex = new JsonMapView((Map<String, Object>) indices.get(".tasks"));
+                assertThat(tasksIndex.get("system"), is(true));
 
-            Map<String, Object> testIndex = (Map<String, Object>) indices.get("test_index_old");
-            assertNotNull(testIndex);
-            assertThat(testIndex.get("system"), is(false));
-        });
+                // If .tasks was created in a 7.x version, it should have an alias on it that we need to make sure got upgraded properly.
+                final String tasksCreatedVersionString = tasksIndex.get("settings.index.version.created");
+                assertThat(tasksCreatedVersionString, notNullValue());
+                final Version tasksCreatedVersion = Version.fromId(Integer.parseInt(tasksCreatedVersionString));
+                if (tasksCreatedVersion.before(SYSTEM_INDEX_ENFORCEMENT_VERSION)) {
+                    // Verify that the alias survived the upgrade
+                    Request getAliasRequest = new Request("GET", "/_alias/test-system-alias");
+                    getAliasRequest.setOptions(expectVersionSpecificWarnings(v -> {
+                        v.current(systemIndexWarning);
+                        v.compatible(systemIndexWarning);
+                    }));
+                    Map<String, Object> aliasResponse = entityAsMap(client().performRequest(getAliasRequest));
+                    assertThat(aliasResponse, hasKey(".tasks"));
+                    assertThat(aliasResponse, hasKey("test_index_reindex"));
+                }
+            });
+        }
     }
 }

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml

@@ -87,9 +87,15 @@
 ---
 "Find a task result record from the old cluster":
   - skip:
-      features: headers
+      features:
+        - headers
+        - allowed_warnings
 
   - do:
+      # We don't require this warning because there's a very brief window during upgrades before the IndexMetaData is upgraded when warnings
+      # may not be emitted. That they do get upgraded is tested specifically by FullClusterRestartIT#testSystemIndexMetadataIsUpgraded().
+      allowed_warnings:
+        - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
       search:
         rest_total_hits_as_int: true
         index: .tasks

qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java

@@ -0,0 +1,173 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.http;
+
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SystemIndexPlugin;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestStatusToXContentListener;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.test.rest.ESRestTestCase.entityAsMap;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.is;
+
+public class SystemIndexRestIT extends HttpSmokeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(SystemIndexTestPlugin.class);
+        return plugins;
+    }
+
+    public void testSystemIndexAccessBlockedByDefault() throws Exception {
+        // create index
+        {
+            Request putDocRequest = new Request("POST", "/_sys_index_test/add_doc/42");
+            Response resp = getRestClient().performRequest(putDocRequest);
+            assertThat(resp.getStatusLine().getStatusCode(), equalTo(201));
+        }
+
+
+        // make sure the system index now exists
+        assertBusy(() -> {
+            Request searchRequest = new Request("GET", "/" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "/_count");
+            searchRequest.setOptions(expectWarnings("this request accesses system indices: [" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME +
+                "], but in a future major version, direct access to system indices will be prevented by default"));
+
+            // Disallow no indices to cause an exception if the flag above doesn't work
+            searchRequest.addParameter("allow_no_indices", "false");
+            searchRequest.setJsonEntity("{\"query\": {\"match\":  {\"some_field\":  \"some_value\"}}}");
+
+            final Response searchResponse = getRestClient().performRequest(searchRequest);
+            assertThat(searchResponse.getStatusLine().getStatusCode(), is(200));
+            Map<String, Object> responseMap = entityAsMap(searchResponse);
+            assertThat(responseMap, hasKey("count"));
+            assertThat(responseMap.get("count"), equalTo(1));
+        });
+
+        // And with a partial wildcard
+        assertDeprecationWarningOnAccess(".test-*", SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+
+        // And with a total wildcard
+        assertDeprecationWarningOnAccess(randomFrom("*", "_all"), SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+
+        // Try to index a doc directly
+        {
+            String expectedWarning = "this request accesses system indices: [" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "], but in a " +
+                "future major version, direct access to system indices will be prevented by default";
+            Request putDocDirectlyRequest = new Request("PUT", "/" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "/_doc/43");
+            putDocDirectlyRequest.setJsonEntity("{\"some_field\":  \"some_other_value\"}");
+            putDocDirectlyRequest.setOptions(expectWarnings(expectedWarning));
+            Response response = getRestClient().performRequest(putDocDirectlyRequest);
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(201));
+        }
+    }
+
+    private void assertDeprecationWarningOnAccess(String queryPattern, String warningIndexName) throws IOException {
+        String expectedWarning = "this request accesses system indices: [" + warningIndexName + "], but in a " +
+            "future major version, direct access to system indices will be prevented by default";
+        Request searchRequest = new Request("GET", "/" + queryPattern + randomFrom("/_count", "/_search"));
+        searchRequest.setJsonEntity("{\"query\": {\"match\":  {\"some_field\":  \"some_value\"}}}");
+        // Disallow no indices to cause an exception if this resolves to zero indices, so that we're sure it resolved the index
+        searchRequest.addParameter("allow_no_indices", "false");
+        searchRequest.setOptions(expectWarnings(expectedWarning));
+
+        Response response = getRestClient().performRequest(searchRequest);
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+    }
+
+    private RequestOptions expectWarnings(String expectedWarning) {
+        final RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
+        builder.setWarningsHandler(w -> w.contains(expectedWarning) == false || w.size() != 1);
+        return builder.build();
+    }
+
+
+    public static class SystemIndexTestPlugin extends Plugin implements SystemIndexPlugin {
+
+        public static final String SYSTEM_INDEX_NAME = ".test-system-idx";
+
+        @Override
+        public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                                                 IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                                                 IndexNameExpressionResolver indexNameExpressionResolver,
+                                                 Supplier<DiscoveryNodes> nodesInCluster) {
+            return org.elasticsearch.common.collect.List.of(new AddDocRestHandler());
+        }
+
+        @Override
+        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System indices for tests"));
+        }
+
+        public static class AddDocRestHandler extends BaseRestHandler {
+            @Override
+            public boolean allowSystemIndexAccessByDefault() {
+                return true;
+            }
+
+            @Override
+            public String getName() {
+                return "system_index_test_doc_adder";
+            }
+
+            @Override
+            public List<Route> routes() {
+                return org.elasticsearch.common.collect.List.of(new Route(RestRequest.Method.POST, "/_sys_index_test/add_doc/{id}"));
+            }
+
+            @Override
+            protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+                IndexRequest indexRequest = new IndexRequest(SYSTEM_INDEX_NAME);
+                indexRequest.id(request.param("id"));
+                indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+                indexRequest.source(org.elasticsearch.common.collect.Map.of("some_field", "some_value"));
+                return channel -> client.index(indexRequest,
+                    new RestStatusToXContentListener<>(channel, r -> r.getLocation(indexRequest.routing())));
+            }
+        }
+    }
+}

server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java

@@ -392,7 +392,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
         internalCluster().coordOnlyNodeClient().admin().indices().flush(flushRequest).actionGet();
 
         clearInterceptedActions();
-        String[] indices = new IndexNameExpressionResolver()
+        String[] indices = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))
                 .concreteIndexNames(client().admin().cluster().prepareState().get().getState(), flushRequest);
         assertIndicesSubset(Arrays.asList(indices), indexShardActions);
     }
@@ -417,7 +417,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
         internalCluster().coordOnlyNodeClient().admin().indices().refresh(refreshRequest).actionGet();
 
         clearInterceptedActions();
-        String[] indices = new IndexNameExpressionResolver()
+        String[] indices = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))
                 .concreteIndexNames(client().admin().cluster().prepareState().get().getState(), refreshRequest);
         assertIndicesSubset(Arrays.asList(indices), indexShardActions);
     }

server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java

@@ -56,6 +56,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -821,7 +822,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
     public void testSnapshotWithDateMath() {
         final String repo = "repo";
 
-        final IndexNameExpressionResolver nameExpressionResolver = new IndexNameExpressionResolver();
+        final IndexNameExpressionResolver nameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
         final String snapshotName = "<snapshot-{now/d}>";
 
         logger.info("-->  creating repository");

server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java

@@ -25,26 +25,38 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
+import java.util.stream.Collectors;
 
 public class TransportGetAliasesAction extends TransportMasterNodeReadAction<GetAliasesRequest, GetAliasesResponse> {
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TransportGetAliasesAction.class);
+
+    private final SystemIndices systemIndices;
 
     @Inject
     public TransportGetAliasesAction(TransportService transportService, ClusterService clusterService,
                                      ThreadPool threadPool, ActionFilters actionFilters,
-                                     IndexNameExpressionResolver indexNameExpressionResolver) {
+                                     IndexNameExpressionResolver indexNameExpressionResolver, SystemIndices systemIndices) {
         super(GetAliasesAction.NAME, transportService, clusterService, threadPool, actionFilters, GetAliasesRequest::new,
             indexNameExpressionResolver);
+        this.systemIndices = systemIndices;
     }
 
     @Override
@@ -55,8 +67,9 @@ public class TransportGetAliasesAction extends TransportMasterNodeReadAction<Get
 
     @Override
     protected ClusterBlockException checkBlock(GetAliasesRequest request, ClusterState state) {
+        // Resolve with system index access since we're just checking blocks
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
-            indexNameExpressionResolver.concreteIndexNames(state, request));
+            indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(state, request));
     }
 
     @Override
@@ -66,16 +79,25 @@ public class TransportGetAliasesAction extends TransportMasterNodeReadAction<Get
 
     @Override
     protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener<GetAliasesResponse> listener) {
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
+        String[] concreteIndices;
+        // Switch to a context which will drop any deprecation warnings, because there may be indices resolved here which are not
+        // returned in the final response. We'll add warnings back later if necessary in checkSystemIndexAccess.
+        try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().newStoredContext(false)) {
+            concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
+        }
+        final boolean systemIndexAccessAllowed = indexNameExpressionResolver.isSystemIndexAccessAllowed();
         ImmutableOpenMap<String, List<AliasMetadata>> aliases = state.metadata().findAliases(request, concreteIndices);
-        listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases)));
+        listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases, state,
+            systemIndexAccessAllowed, systemIndices)));
     }
 
     /**
      * Fills alias result with empty entries for requested indices when no specific aliases were requested.
      */
     static ImmutableOpenMap<String, List<AliasMetadata>> postProcess(GetAliasesRequest request, String[] concreteIndices,
-                                                                     ImmutableOpenMap<String, List<AliasMetadata>> aliases) {
+                                                                     ImmutableOpenMap<String, List<AliasMetadata>> aliases,
+                                                                     ClusterState state, boolean systemIndexAccessAllowed,
+                                                                     SystemIndices systemIndices) {
         boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0;
         ImmutableOpenMap.Builder<String, List<AliasMetadata>> mapBuilder = ImmutableOpenMap.builder(aliases);
         for (String index : concreteIndices) {
@@ -84,7 +106,40 @@ public class TransportGetAliasesAction extends TransportMasterNodeReadAction<Get
                 assert previous == null;
             }
         }
-        return mapBuilder.build();
+        final ImmutableOpenMap<String, List<AliasMetadata>> finalResponse = mapBuilder.build();
+        if (systemIndexAccessAllowed == false) {
+            checkSystemIndexAccess(request, systemIndices, state, finalResponse);
+        }
+        return finalResponse;
     }
 
+    private static void checkSystemIndexAccess(GetAliasesRequest request, SystemIndices systemIndices, ClusterState state,
+                                               ImmutableOpenMap<String, List<AliasMetadata>> aliasesMap) {
+        List<String> systemIndicesNames = new ArrayList<>();
+        for (Iterator<String> it = aliasesMap.keysIt(); it.hasNext(); ) {
+            String indexName = it.next();
+            IndexMetadata index = state.metadata().index(indexName);
+            if (index != null && index.isSystem()) {
+                systemIndicesNames.add(indexName);
+            }
+        }
+        if (systemIndicesNames.isEmpty() == false) {
+            deprecationLogger.deprecate("open_system_index_access",
+                "this request accesses system indices: {}, but in a future major version, direct access to system " +
+                    "indices will be prevented by default", systemIndicesNames);
+        } else {
+            checkSystemAliasAccess(request, systemIndices);
+        }
+    }
+
+    private static void checkSystemAliasAccess(GetAliasesRequest request, SystemIndices systemIndices) {
+        final List<String> systemAliases = Arrays.stream(request.aliases())
+            .filter(alias -> systemIndices.isSystemIndex(alias))
+            .collect(Collectors.toList());
+        if (systemAliases.isEmpty() == false) {
+            deprecationLogger.deprecate("open_system_alias_access",
+                "this request accesses aliases with names reserved for system indices: {}, but in a future major version, direct" +
+                    "access to system indices and their aliases will not be allowed", systemAliases);
+        }
+    }
 }

server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java

@@ -220,6 +220,17 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
      */
     protected abstract ClusterBlockException checkRequestBlock(ClusterState state, Request request, String[] concreteIndices);
 
+    /**
+     * Resolves a list of concrete index names. Override this if index names should be resolved differently than normal.
+     *
+     * @param clusterState the cluster state
+     * @param request the underlying request
+     * @return a list of concrete index names that this action should operate on
+     */
+    protected String[] resolveConcreteIndexNames(ClusterState clusterState, Request request) {
+        return indexNameExpressionResolver.concreteIndexNames(clusterState, request);
+    }
+
     @Override
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
         new AsyncAction(task, request, listener).start();
@@ -249,7 +260,7 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
                 throw globalBlockException;
             }
 
-            String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
+            String[] concreteIndices = resolveConcreteIndexNames(clusterState, request);
             ClusterBlockException requestBlockException = checkRequestBlock(clusterState, request, concreteIndices);
             if (requestBlockException != null) {
                 throw requestBlockException;

server/src/main/java/org/elasticsearch/cluster/ClusterModule.java

@@ -23,10 +23,10 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.ComponentTemplateMetadata;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.DataStreamMetadata;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexAliasesService;
@@ -69,6 +69,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.ingest.IngestMetadata;
@@ -111,13 +112,13 @@ public class ClusterModule extends AbstractModule {
     final ShardsAllocator shardsAllocator;
 
     public ClusterModule(Settings settings, ClusterService clusterService, List<ClusterPlugin> clusterPlugins,
-                         ClusterInfoService clusterInfoService, SnapshotsInfoService snapshotsInfoService) {
+                         ClusterInfoService clusterInfoService, SnapshotsInfoService snapshotsInfoService, ThreadContext threadContext) {
         this.clusterPlugins = clusterPlugins;
         this.deciderList = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins);
         this.allocationDeciders = new AllocationDeciders(deciderList);
         this.shardsAllocator = createShardsAllocator(settings, clusterService.getClusterSettings(), clusterPlugins);
         this.clusterService = clusterService;
-        this.indexNameExpressionResolver = new IndexNameExpressionResolver();
+        this.indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext);
         this.allocationService = new AllocationService(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService);
     }
 

server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java

@@ -81,7 +81,7 @@ public interface IndexAbstraction {
     boolean isHidden();
 
     /**
-     * @return whether this index abstraction is hidden or not
+     * @return whether this index abstraction should be treated as a system index or not
      */
     boolean isSystem();
 

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -20,18 +20,22 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -59,20 +63,38 @@ import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
 public class IndexNameExpressionResolver {
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IndexNameExpressionResolver.class);
 
     public static final String EXCLUDED_DATA_STREAMS_KEY = "es.excluded_ds";
+    public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed";
+    public static final Version SYSTEM_INDEX_ENFORCEMENT_VERSION = Version.V_7_10_0;
 
     private final DateMathExpressionResolver dateMathExpressionResolver = new DateMathExpressionResolver();
     private final WildcardExpressionResolver wildcardExpressionResolver = new WildcardExpressionResolver();
     private final List<ExpressionResolver> expressionResolvers =
         org.elasticsearch.common.collect.List.of(dateMathExpressionResolver, wildcardExpressionResolver);
 
+    private final ThreadContext threadContext;
+
+    public IndexNameExpressionResolver(ThreadContext threadContext) {
+        this.threadContext = Objects.requireNonNull(threadContext, "Thread Context must not be null");
+    }
+
     /**
      * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options
      * are encapsulated in the specified request.
      */
     public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
-        Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams());
+        Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams(),
+            isSystemIndexAccessAllowed());
+        return concreteIndexNames(context, request.indices());
+    }
+
+    /**
+     * Same as {@link #concreteIndexNames(ClusterState, IndicesRequest)}, but access to system indices is always allowed.
+     */
+    public String[] concreteIndexNamesWithSystemIndexAccess(ClusterState state, IndicesRequest request) {
+        Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams(), true);
         return concreteIndexNames(context, request.indices());
     }
 
@@ -81,7 +103,8 @@ public class IndexNameExpressionResolver {
      * are encapsulated in the specified request and resolves data streams.
      */
     public Index[] concreteIndices(ClusterState state, IndicesRequest request) {
-        Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams());
+        Context context = new Context(state, request.indicesOptions(), false, false, request.includeDataStreams(),
+            isSystemIndexAccessAllowed());
         return concreteIndices(context, request.indices());
     }
 
@@ -99,22 +122,23 @@ public class IndexNameExpressionResolver {
      * indices options in the context don't allow such a case.
      */
     public String[] concreteIndexNames(ClusterState state, IndicesOptions options, String... indexExpressions) {
-        Context context = new Context(state, options);
+        Context context = new Context(state, options, isSystemIndexAccessAllowed());
         return concreteIndexNames(context, indexExpressions);
     }
 
     public String[] concreteIndexNames(ClusterState state, IndicesOptions options, boolean includeDataStreams, String... indexExpressions) {
-        Context context = new Context(state, options, false, false, includeDataStreams);
+        Context context = new Context(state, options, false, false, includeDataStreams, isSystemIndexAccessAllowed());
         return concreteIndexNames(context, indexExpressions);
     }
 
     public String[] concreteIndexNames(ClusterState state, IndicesOptions options, IndicesRequest request) {
-        Context context = new Context(state, options, false, false, request.includeDataStreams());
+        Context context = new Context(state, options, false, false, request.includeDataStreams(), isSystemIndexAccessAllowed());
         return concreteIndexNames(context, request.indices());
     }
 
     public List<String> dataStreamNames(ClusterState state, IndicesOptions options, String... indexExpressions) {
-        Context context = new Context(state, options, false, false, true, true);
+        // Allow system index access - they'll be filtered out below as there's no such thing (yet) as system data streams
+        Context context = new Context(state, options, false, false, true, true, true);
         if (indexExpressions == null || indexExpressions.length == 0) {
             indexExpressions = new String[]{"*"};
         }
@@ -146,7 +170,8 @@ public class IndexNameExpressionResolver {
     }
 
     public Index[] concreteIndices(ClusterState state, IndicesOptions options, boolean includeDataStreams, String... indexExpressions) {
-        Context context = new Context(state, options, false, false, includeDataStreams);
+        Context context = new Context(state, options, false, false, includeDataStreams,
+            isSystemIndexAccessAllowed());
         return concreteIndices(context, indexExpressions);
     }
 
@@ -163,7 +188,8 @@ public class IndexNameExpressionResolver {
      * indices options in the context don't allow such a case.
      */
     public Index[] concreteIndices(ClusterState state, IndicesRequest request, long startTime) {
-        Context context = new Context(state, request.indicesOptions(), startTime, false, false, request.includeDataStreams(), false);
+        Context context = new Context(state, request.indicesOptions(), startTime, false, false, request.includeDataStreams(), false,
+            isSystemIndexAccessAllowed());
         return concreteIndices(context, request.indices());
     }
 
@@ -283,9 +309,26 @@ public class IndexNameExpressionResolver {
             }
             throw infe;
         }
+        checkSystemIndexAccess(context, metadata, concreteIndices, indexExpressions);
         return concreteIndices.toArray(new Index[concreteIndices.size()]);
     }
 
+    private void checkSystemIndexAccess(Context context, Metadata metadata, Set<Index> concreteIndices, String[] originalPatterns) {
+        if (context.isSystemIndexAccessAllowed() == false) {
+            final List<String> resolvedSystemIndices = concreteIndices.stream()
+                .map(metadata::index)
+                .filter(IndexMetadata::isSystem)
+                .map(i -> i.getIndex().getName())
+                .sorted() // reliable order for testing
+                .collect(Collectors.toList());
+            if (resolvedSystemIndices.isEmpty() == false) {
+                deprecationLogger.deprecate("open_system_index_access",
+                    "this request accesses system indices: {}, but in a future major version, direct access to system " +
+                        "indices will be prevented by default", resolvedSystemIndices);
+            }
+        }
+    }
+
     private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, IndexMetadata index) {
         if (index.getState() == IndexMetadata.State.CLOSE) {
             if (options.forbidClosedIndices() && options.ignoreUnavailable() == false) {
@@ -370,7 +413,7 @@ public class IndexNameExpressionResolver {
             options.allowAliasesToMultipleIndices(), options.forbidClosedIndices(), options.ignoreAliases(),
             options.ignoreThrottled());
 
-        Context context = new Context(state, combinedOptions, false, true, includeDataStreams);
+        Context context = new Context(state, combinedOptions, false, true, includeDataStreams, isSystemIndexAccessAllowed());
         Index[] indices = concreteIndices(context, index);
         if (allowNoIndices && indices.length == 0) {
             return null;
@@ -387,7 +430,7 @@ public class IndexNameExpressionResolver {
      *         If the data stream, index or alias contains date math then that is resolved too.
      */
     public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) {
-        Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true);
+        Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true, isSystemIndexAccessAllowed());
         String resolvedAliasOrIndex = dateMathExpressionResolver.resolveExpression(indexAbstraction, context);
         return state.metadata().getIndicesLookup().containsKey(resolvedAliasOrIndex);
     }
@@ -398,14 +441,14 @@ public class IndexNameExpressionResolver {
     public String resolveDateMathExpression(String dateExpression) {
         // The data math expression resolver doesn't rely on cluster state or indices options, because
         // it just resolves the date math to an actual date.
-        return dateMathExpressionResolver.resolveExpression(dateExpression, new Context(null, null));
+        return dateMathExpressionResolver.resolveExpression(dateExpression, new Context(null, null, isSystemIndexAccessAllowed()));
     }
 
     /**
      * Resolve an array of expressions to the set of indices and aliases that these expressions match.
      */
     public Set<String> resolveExpressions(ClusterState state, String... expressions) {
-        Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false, true);
+        Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false, true, isSystemIndexAccessAllowed());
         List<String> resolvedExpressions = Arrays.asList(expressions);
         for (ExpressionResolver expressionResolver : expressionResolvers) {
             resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
@@ -499,7 +542,7 @@ public class IndexNameExpressionResolver {
      */
     public Map<String, Set<String>> resolveSearchRouting(ClusterState state, @Nullable String routing, String... expressions) {
         List<String> resolvedExpressions = expressions != null ? Arrays.asList(expressions) : Collections.emptyList();
-        Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true);
+        Context context = new Context(state, IndicesOptions.lenientExpandOpen(), false, false, true, isSystemIndexAccessAllowed());
         for (ExpressionResolver expressionResolver : expressionResolvers) {
             resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
         }
@@ -651,6 +694,15 @@ public class IndexNameExpressionResolver {
         return false;
     }
 
+    /**
+     * Determines whether or not system index access should be allowed in the current context.
+     *
+     * @return True if system index access should be allowed, false otherwise.
+     */
+    public boolean isSystemIndexAccessAllowed() {
+        return Booleans.parseBoolean(threadContext.getHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY), true);
+    }
+
     public static class Context {
 
         private final ClusterState state;
@@ -660,27 +712,30 @@ public class IndexNameExpressionResolver {
         private final boolean resolveToWriteIndex;
         private final boolean includeDataStreams;
         private final boolean preserveDataStreams;
+        private final boolean isSystemIndexAccessAllowed;
 
-        Context(ClusterState state, IndicesOptions options) {
-            this(state, options, System.currentTimeMillis());
+        Context(ClusterState state, IndicesOptions options, boolean isSystemIndexAccessAllowed) {
+            this(state, options, System.currentTimeMillis(), isSystemIndexAccessAllowed);
         }
 
         Context(ClusterState state, IndicesOptions options, boolean preserveAliases, boolean resolveToWriteIndex,
-                boolean includeDataStreams) {
-            this(state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex, includeDataStreams, false);
+                boolean includeDataStreams, boolean isSystemIndexAccessAllowed) {
+            this(state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex, includeDataStreams, false,
+                isSystemIndexAccessAllowed);
         }
 
         Context(ClusterState state, IndicesOptions options, boolean preserveAliases, boolean resolveToWriteIndex,
-                boolean includeDataStreams, boolean preserveDataStreams) {
-            this(state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex, includeDataStreams, preserveDataStreams);
+                boolean includeDataStreams, boolean preserveDataStreams, boolean isSystemIndexAccessAllowed) {
+            this(state, options, System.currentTimeMillis(), preserveAliases, resolveToWriteIndex, includeDataStreams, preserveDataStreams,
+                isSystemIndexAccessAllowed);
         }
 
-        Context(ClusterState state, IndicesOptions options, long startTime) {
-           this(state, options, startTime, false, false, false, false);
+        Context(ClusterState state, IndicesOptions options, long startTime, boolean isSystemIndexAccessAllowed) {
+           this(state, options, startTime, false, false, false, false, isSystemIndexAccessAllowed);
         }
 
         protected Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases, boolean resolveToWriteIndex,
-                          boolean includeDataStreams, boolean preserveDataStreams) {
+                          boolean includeDataStreams, boolean preserveDataStreams, boolean isSystemIndexAccessAllowed) {
             this.state = state;
             this.options = options;
             this.startTime = startTime;
@@ -688,6 +743,7 @@ public class IndexNameExpressionResolver {
             this.resolveToWriteIndex = resolveToWriteIndex;
             this.includeDataStreams = includeDataStreams;
             this.preserveDataStreams = preserveDataStreams;
+            this.isSystemIndexAccessAllowed = isSystemIndexAccessAllowed;
         }
 
         public ClusterState getState() {
@@ -726,6 +782,13 @@ public class IndexNameExpressionResolver {
         public boolean isPreserveDataStreams() {
             return preserveDataStreams;
         }
+
+        /**
+         * Used to determine if it is allowed to access system indices in this context (e.g. for this request).
+         */
+        public boolean isSystemIndexAccessAllowed() {
+            return isSystemIndexAccessAllowed;
+        }
     }
 
     private interface ExpressionResolver {

server/src/main/java/org/elasticsearch/indices/SystemIndices.java

@@ -50,7 +50,6 @@ import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX;
  * to reduce the locations within the code that need to deal with {@link SystemIndexDescriptor}s.
  */
 public class SystemIndices {
-
     private static final Map<String, Collection<SystemIndexDescriptor>> SERVER_SYSTEM_INDEX_DESCRIPTORS = singletonMap(
         TaskResultsService.class.getName(), singletonList(new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index"))
     );

server/src/main/java/org/elasticsearch/node/Node.java

@@ -425,7 +425,7 @@ public class Node implements Closeable {
             final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService(settings, clusterService,
                 repositoriesServiceReference::get, rerouteServiceReference::get);
             final ClusterModule clusterModule = new ClusterModule(settings, clusterService, clusterPlugins, clusterInfoService,
-                snapshotsInfoService);
+                snapshotsInfoService, threadPool.getThreadContext());
             modules.add(clusterModule);
             IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
             modules.add(indicesModule);

server/src/main/java/org/elasticsearch/rest/RestController.java

@@ -54,6 +54,7 @@ import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY;
 import static org.elasticsearch.rest.BytesRestResponse.TEXT_CONTENT_TYPE;
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
@@ -65,6 +66,7 @@ public class RestController implements HttpServerTransport.Dispatcher {
 
     private static final Logger logger = LogManager.getLogger(RestController.class);
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestController.class);
+    private static final String ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER = "X-elastic-product-origin";
 
     private static final BytesReference FAVICON_RESPONSE;
 
@@ -246,6 +248,13 @@ public class RestController implements HttpServerTransport.Dispatcher {
             if (handler.allowsUnsafeBuffers() == false) {
                 request.ensureSafeBuffers();
             }
+            if (handler.allowSystemIndexAccessByDefault() == false && request.header(ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER) == null) {
+                // The ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER indicates that the request is coming from an Elastic product with a plan
+                // to move away from direct access to system indices, and thus deprecation warnings should not be emitted.
+                // This header is intended for internal use only.
+                client.threadPool().getThreadContext().putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.FALSE.toString());
+            }
+
             handler.handleRequest(request, responseChannel, client);
         } catch (Exception e) {
             responseChannel.sendResponse(new BytesRestResponse(responseChannel, e));

server/src/main/java/org/elasticsearch/rest/RestHandler.java

@@ -90,6 +90,15 @@ public interface RestHandler {
         return Collections.emptyList();
     }
 
+
+    /**
+     * Controls whether requests handled by this class are allowed to to access system indices by default.
+     * @return {@code true} if requests handled by this class should be allowed to access system indices.
+     */
+    default boolean allowSystemIndexAccessByDefault() {
+        return false;
+    }
+
     class Route {
 
         private final String path;

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java

@@ -57,6 +57,11 @@ public class RestClusterAllocationExplainAction extends BaseRestHandler {
         return "cluster_allocation_explain_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClusterAllocationExplainRequest req;

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java

@@ -54,6 +54,11 @@ public class RestClusterHealthAction extends BaseRestHandler {
         return "cluster_health_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final ClusterHealthRequest clusterHealthRequest = fromRequest(request);

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java

@@ -71,6 +71,11 @@ public class RestClusterRerouteAction extends BaseRestHandler {
         return "cluster_reroute_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClusterRerouteRequest clusterRerouteRequest = createRequest(request);

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java

@@ -71,6 +71,11 @@ public class RestClusterStateAction extends BaseRestHandler {
             new Route(GET, "/_cluster/state/{metric}/{indices}")));
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final ClusterStateRequest clusterStateRequest = Requests.clusterStateRequest();

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java

@@ -75,7 +75,8 @@ public class RestGetAliasesAction extends BaseRestHandler {
     }
 
     static RestResponse buildRestResponse(boolean aliasesExplicitlyRequested, String[] requestedAliases,
-            ImmutableOpenMap<String, List<AliasMetadata>> responseAliasMap, XContentBuilder builder) throws Exception {
+                                          ImmutableOpenMap<String, List<AliasMetadata>> responseAliasMap,
+                                          XContentBuilder builder) throws Exception {
         final Set<String> indicesToDisplay = new HashSet<>();
         final Set<String> returnedAliasNames = new HashSet<>();
         for (final ObjectObjectCursor<String, List<AliasMetadata>> cursor : responseAliasMap) {

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java

@@ -57,6 +57,11 @@ public class RestIndicesShardStoresAction extends BaseRestHandler {
         return "indices_shard_stores_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndicesShardStoresRequest indicesShardStoresRequest = new IndicesShardStoresRequest(

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java

@@ -59,6 +59,11 @@ public class RestIndicesStatsAction extends BaseRestHandler {
         return "indices_stats_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     static final Map<String, Consumer<IndicesStatsRequest>> METRICS;
 
     static {

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java

@@ -51,6 +51,11 @@ public class RestRecoveryAction extends BaseRestHandler {
         return "recovery_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
 

server/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java

@@ -50,6 +50,11 @@ public class RestAliasAction extends AbstractCatAction {
         return "cat_alias_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final GetAliasesRequest getAliasesRequest = request.hasParam("alias") ?

server/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java

@@ -45,11 +45,17 @@ public class RestHealthAction extends AbstractCatAction {
         return "cat_health_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     protected void documentation(StringBuilder sb) {
         sb.append("/_cat/health\n");
     }
 
+
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest();

server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java

@@ -82,6 +82,11 @@ public class RestIndicesAction extends AbstractCatAction {
         return "cat_indices_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     protected void documentation(StringBuilder sb) {
         sb.append("/_cat/indices\n");

server/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java

@@ -57,6 +57,11 @@ public class RestSegmentsAction extends AbstractCatAction {
         return "cat_segments_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));

server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java

@@ -73,6 +73,11 @@ public class RestShardsAction extends AbstractCatAction {
         return "cat_shards_action";
     }
 
+    @Override
+    public boolean allowSystemIndexAccessByDefault() {
+        return true;
+    }
+
     @Override
     protected void documentation(StringBuilder sb) {
         sb.append("/_cat/shards\n");

server/src/test/java/org/elasticsearch/action/ActionModuleTests.java

@@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
 import org.elasticsearch.rest.RestChannel;
@@ -107,9 +108,10 @@ public class ActionModuleTests extends ESTestCase {
     public void testSetupRestHandlerContainsKnownBuiltin() {
         SettingsModule settings = new SettingsModule(Settings.EMPTY);
         UsageService usageService = new UsageService();
-        ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
-                settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
-                null, usageService, null);
+        ActionModule actionModule = new ActionModule(false, settings.getSettings(),
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), settings.getIndexScopedSettings(),
+            settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
+            null, usageService, null);
         actionModule.initRestHandlers(null);
         // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
         Exception e = expectThrows(IllegalArgumentException.class, () ->
@@ -146,9 +148,10 @@ public class ActionModuleTests extends ESTestCase {
         ThreadPool threadPool = new TestThreadPool(getTestName());
         try {
             UsageService usageService = new UsageService();
-            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
-                    settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
-                    singletonList(dupsMainAction), null, null, usageService, null);
+            ActionModule actionModule = new ActionModule(false, settings.getSettings(),
+                new IndexNameExpressionResolver(threadPool.getThreadContext()), settings.getIndexScopedSettings(),
+                settings.getClusterSettings(), settings.getSettingsFilter(), threadPool, singletonList(dupsMainAction),
+                null, null, usageService, null);
             Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
             assertThat(e.getMessage(), startsWith("Cannot replace existing handler for [/] for method: GET"));
         } finally {
@@ -180,9 +183,10 @@ public class ActionModuleTests extends ESTestCase {
         ThreadPool threadPool = new TestThreadPool(getTestName());
         try {
             UsageService usageService = new UsageService();
-            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
-                    settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
-                    singletonList(registersFakeHandler), null, null, usageService, null);
+            ActionModule actionModule = new ActionModule(false, settings.getSettings(),
+                new IndexNameExpressionResolver(threadPool.getThreadContext()), settings.getIndexScopedSettings(),
+                settings.getClusterSettings(), settings.getSettingsFilter(), threadPool, singletonList(registersFakeHandler),
+                null, null, usageService, null);
             actionModule.initRestHandlers(null);
             // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
             Exception e = expectThrows(IllegalArgumentException.class, () ->

server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java

@@ -41,6 +41,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -132,7 +133,7 @@ public class TransportAddVotingConfigExclusionsActionTests extends ESTestCase {
         clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
 
         new TransportAddVotingConfigExclusionsAction(nodeSettings, clusterSettings, transportService, clusterService, threadPool,
-            new ActionFilters(emptySet()), new IndexNameExpressionResolver()); // registers action
+            new ActionFilters(emptySet()), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))); // registers action
 
         transportService.start();
         transportService.acceptIncomingRequests();

server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsActionTests.java

@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -95,7 +96,7 @@ public class TransportClearVotingConfigExclusionsActionTests extends ESTestCase
             TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundTransportAddress -> localNode, null, emptySet());
 
         new TransportClearVotingConfigExclusionsAction(transportService, clusterService, threadPool, new ActionFilters(emptySet()),
-            new IndexNameExpressionResolver()); // registers action
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))); // registers action
 
         transportService.start();
         transportService.acceptIncomingRequests();

server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java

@@ -18,8 +18,14 @@
  */
 package org.elasticsearch.action.admin.indices.alias.get;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
@@ -28,6 +34,7 @@ import java.util.List;
 import static org.hamcrest.Matchers.equalTo;
 
 public class TransportGetAliasesActionTests extends ESTestCase {
+    private final SystemIndices EMPTY_SYSTEM_INDICES = new SystemIndices(Collections.emptyMap());
 
     public void testPostProcess() {
         GetAliasesRequest request = new GetAliasesRequest();
@@ -35,7 +42,8 @@ public class TransportGetAliasesActionTests extends ESTestCase {
             .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build()))
             .build();
         ImmutableOpenMap<String, List<AliasMetadata>> result =
-            TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases);
+            TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, false,
+                EMPTY_SYSTEM_INDICES);
         assertThat(result.size(), equalTo(3));
         assertThat(result.get("a").size(), equalTo(0));
         assertThat(result.get("b").size(), equalTo(1));
@@ -46,7 +54,8 @@ public class TransportGetAliasesActionTests extends ESTestCase {
         aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
             .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build()))
             .build();
-        result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases);
+        result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, false,
+            EMPTY_SYSTEM_INDICES);
         assertThat(result.size(), equalTo(3));
         assertThat(result.get("a").size(), equalTo(0));
         assertThat(result.get("b").size(), equalTo(1));
@@ -56,9 +65,129 @@ public class TransportGetAliasesActionTests extends ESTestCase {
         aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
             .fPut("b", Collections.singletonList(new AliasMetadata.Builder("y").build()))
             .build();
-        result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases);
+        result = TransportGetAliasesAction.postProcess(request, new String[]{"a", "b", "c"}, aliases, ClusterState.EMPTY_STATE, false,
+            EMPTY_SYSTEM_INDICES);
         assertThat(result.size(), equalTo(1));
         assertThat(result.get("b").size(), equalTo(1));
     }
 
+    public void testDeprecationWarningEmittedForTotalWildcard() {
+        ClusterState state = systemIndexTestClusterState();
+
+        GetAliasesRequest request = new GetAliasesRequest();
+        ImmutableOpenMap<String, List<AliasMetadata>> aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
+            .fPut(".b", Collections.singletonList(new AliasMetadata.Builder(".y").build()))
+            .fPut("c", Collections.singletonList(new AliasMetadata.Builder("d").build()))
+            .build();
+        final String[] concreteIndices = {"a", ".b", "c"};
+        assertEquals(state.metadata().findAliases(request, concreteIndices), aliases);
+        ImmutableOpenMap<String, List<AliasMetadata>> result =
+            TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, false, EMPTY_SYSTEM_INDICES);
+        assertThat(result.size(), equalTo(3));
+        assertThat(result.get("a").size(), equalTo(0));
+        assertThat(result.get(".b").size(), equalTo(1));
+        assertThat(result.get("c").size(), equalTo(1));
+        assertWarnings("this request accesses system indices: [.b], but in a future major version, direct access to system " +
+            "indices will be prevented by default");
+    }
+
+    public void testDeprecationWarningEmittedWhenSystemIndexIsRequested() {
+        ClusterState state = systemIndexTestClusterState();
+
+        GetAliasesRequest request = new GetAliasesRequest();
+        request.indices(".b");
+        ImmutableOpenMap<String, List<AliasMetadata>> aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
+            .fPut(".b", Collections.singletonList(new AliasMetadata.Builder(".y").build()))
+            .build();
+        final String[] concreteIndices = {".b"};
+        assertEquals(state.metadata().findAliases(request, concreteIndices), aliases);
+        ImmutableOpenMap<String, List<AliasMetadata>> result =
+            TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, false, EMPTY_SYSTEM_INDICES);
+        assertThat(result.size(), equalTo(1));
+        assertThat(result.get(".b").size(), equalTo(1));
+        assertWarnings("this request accesses system indices: [.b], but in a future major version, direct access to system " +
+            "indices will be prevented by default");
+    }
+
+    public void testDeprecationWarningEmittedWhenSystemIndexIsRequestedByAlias() {
+        ClusterState state = systemIndexTestClusterState();
+
+        GetAliasesRequest request = new GetAliasesRequest(".y");
+        ImmutableOpenMap<String, List<AliasMetadata>> aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
+            .fPut(".b", Collections.singletonList(new AliasMetadata.Builder(".y").build()))
+            .build();
+        final String[] concreteIndices = {"a", ".b", "c"};
+        assertEquals(state.metadata().findAliases(request, concreteIndices), aliases);
+        ImmutableOpenMap<String, List<AliasMetadata>> result =
+            TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, false, EMPTY_SYSTEM_INDICES);
+        assertThat(result.size(), equalTo(1));
+        assertThat(result.get(".b").size(), equalTo(1));
+        assertWarnings("this request accesses system indices: [.b], but in a future major version, direct access to system " +
+            "indices will be prevented by default");
+    }
+
+    public void testDeprecationWarningNotEmittedWhenSystemAccessAllowed() {
+        ClusterState state = systemIndexTestClusterState();
+
+        GetAliasesRequest request = new GetAliasesRequest(".y");
+        ImmutableOpenMap<String, List<AliasMetadata>> aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
+            .fPut(".b", Collections.singletonList(new AliasMetadata.Builder(".y").build()))
+            .build();
+        final String[] concreteIndices = {"a", ".b", "c"};
+        assertEquals(state.metadata().findAliases(request, concreteIndices), aliases);
+        ImmutableOpenMap<String, List<AliasMetadata>> result =
+            TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, true, EMPTY_SYSTEM_INDICES);
+        assertThat(result.size(), equalTo(1));
+        assertThat(result.get(".b").size(), equalTo(1));
+    }
+
+    /**
+     * Ensures that deprecation warnings are not emitted when
+     */
+    public void testDeprecationWarningNotEmittedWhenOnlyNonsystemIndexRequested() {
+        ClusterState state = systemIndexTestClusterState();
+
+        GetAliasesRequest request = new GetAliasesRequest();
+        request.indices("c");
+        ImmutableOpenMap<String, List<AliasMetadata>> aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
+            .fPut("c", Collections.singletonList(new AliasMetadata.Builder("d").build()))
+            .build();
+        final String[] concreteIndices = {"c"};
+        assertEquals(state.metadata().findAliases(request, concreteIndices), aliases);
+        ImmutableOpenMap<String, List<AliasMetadata>> result =
+            TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, false, EMPTY_SYSTEM_INDICES);
+        assertThat(result.size(), equalTo(1));
+        assertThat(result.get("c").size(), equalTo(1));
+    }
+
+    public void testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemPattern() {
+        ClusterState state = systemIndexTestClusterState();
+        SystemIndices systemIndices = new SystemIndices(Collections.singletonMap(this.getTestName(),
+            Collections.singletonList(new SystemIndexDescriptor(".y", "an index that doesn't exist"))));
+
+        GetAliasesRequest request = new GetAliasesRequest(".y");
+        ImmutableOpenMap<String, List<AliasMetadata>> aliases = ImmutableOpenMap.<String, List<AliasMetadata>>builder()
+            .build();
+        final String[] concreteIndices = {};
+        assertEquals(state.metadata().findAliases(request, concreteIndices), aliases);
+        ImmutableOpenMap<String, List<AliasMetadata>> result =
+            TransportGetAliasesAction.postProcess(request, concreteIndices, aliases, state, false, systemIndices);
+        assertThat(result.size(), equalTo(0));
+        assertWarnings("this request accesses aliases with names reserved for system indices: [.y], but in a future major version, direct" +
+            "access to system indices and their aliases will not be allowed");
+    }
+
+    public ClusterState systemIndexTestClusterState() {
+        return ClusterState.builder(ClusterState.EMPTY_STATE)
+            .metadata(Metadata.builder()
+                .put(IndexMetadata.builder("a").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
+                .put(IndexMetadata.builder(".b").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)
+                    .system(true).putAlias(AliasMetadata.builder(".y")))
+                .put(IndexMetadata.builder("c").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)
+                    .putAlias(AliasMetadata.builder("d")))
+                .build())
+            .build();
+    }
+
+
 }

server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java

@@ -68,6 +68,9 @@ public class RestForceMergeActionTests extends RestActionTestCase {
             .withParams(params)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings("setting only_expunge_deletes and max_num_segments at the same time is deprecated " +
             "and will be rejected in a future version");

server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexActionTests.java

@@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -121,6 +122,10 @@ public class GetIndexActionTests extends ESSingleNodeTestCase {
     }
 
     static class Resolver extends IndexNameExpressionResolver {
+        Resolver() {
+            super(new ThreadContext(Settings.EMPTY));
+        }
+
         @Override
         public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();

server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java

@@ -30,6 +30,8 @@ import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -168,7 +170,8 @@ public class PutMappingRequestTests extends ESTestCase {
             tuple("alias2", org.elasticsearch.common.collect.List.of(tuple("index2", false), tuple("index3", true)))
         ));
         PutMappingRequest request = new PutMappingRequest().indices("foo", "alias1", "alias2").writeIndexOnly(true);
-        Index[] indices = TransportPutMappingAction.resolveIndices(cs, request, new IndexNameExpressionResolver());
+        Index[] indices = TransportPutMappingAction.resolveIndices(cs, request,
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)));
         List<String> indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList());
         IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo");
         // should resolve the data stream and each alias to their respective write indices
@@ -189,7 +192,8 @@ public class PutMappingRequestTests extends ESTestCase {
             tuple("alias2", org.elasticsearch.common.collect.List.of(tuple("index2", false), tuple("index3", true)))
         ));
         PutMappingRequest request = new PutMappingRequest().indices("foo", "alias1", "alias2");
-        Index[] indices = TransportPutMappingAction.resolveIndices(cs, request, new IndexNameExpressionResolver());
+        Index[] indices = TransportPutMappingAction.resolveIndices(cs, request,
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)));
         List<String> indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList());
         IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo");
         List<String> expectedIndices = expectedDs.getIndices().stream().map(im -> im.getIndex().getName()).collect(Collectors.toList());
@@ -212,7 +216,8 @@ public class PutMappingRequestTests extends ESTestCase {
             tuple("alias2", org.elasticsearch.common.collect.List.of(tuple("index2", false), tuple("index3", true)))
         ));
         PutMappingRequest request = new PutMappingRequest().indices("foo", "index3").writeIndexOnly(true);
-        Index[] indices = TransportPutMappingAction.resolveIndices(cs, request, new IndexNameExpressionResolver());
+        Index[] indices = TransportPutMappingAction.resolveIndices(cs, request,
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)));
         List<String> indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList());
         IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo");
         List<String> expectedIndices = expectedDs.getIndices().stream().map(im -> im.getIndex().getName()).collect(Collectors.toList());
@@ -236,7 +241,8 @@ public class PutMappingRequestTests extends ESTestCase {
         ));
         PutMappingRequest request = new PutMappingRequest().indices("*").writeIndexOnly(true);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> TransportPutMappingAction.resolveIndices(cs2, request, new IndexNameExpressionResolver()));
+            () -> TransportPutMappingAction.resolveIndices(cs2, request,
+                new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))));
         assertThat(e.getMessage(), containsString("The index expression [*] and options provided did not point to a single write-index"));
     }
 
@@ -255,7 +261,8 @@ public class PutMappingRequestTests extends ESTestCase {
         ));
         PutMappingRequest request = new PutMappingRequest().indices("alias2").writeIndexOnly(true);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> TransportPutMappingAction.resolveIndices(cs2, request, new IndexNameExpressionResolver()));
+            () -> TransportPutMappingAction.resolveIndices(cs2, request,
+                new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY))));
         assertThat(e.getMessage(), containsString("no write index is defined for alias [alias2]"));
     }
 

server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java

@@ -34,6 +34,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -69,7 +70,8 @@ public class ResolveIndexTests extends ESTestCase {
     };
 
     private Metadata metadata = buildMetadata(dataStreams, indices);
-    private IndexAbstractionResolver resolver = new IndexAbstractionResolver(new IndexNameExpressionResolver());
+    private IndexAbstractionResolver resolver = new IndexAbstractionResolver(
+        new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)));
 
     public void testResolveStarWithDefaultOptions() {
         String[] names = new String[] {"*"};

server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

@@ -51,6 +51,7 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
@@ -297,7 +298,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
 
     public void testGenerateRolloverIndexName() {
         String invalidIndexName = randomAlphaOfLength(10) + "A";
-        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
         expectThrows(IllegalArgumentException.class, () ->
             MetadataRolloverService.generateRolloverIndexName(invalidIndexName, indexNameExpressionResolver));
         int num = randomIntBetween(0, 100);

server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java

@@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
@@ -129,6 +130,10 @@ public class GetSettingsActionTests extends ESTestCase {
     }
 
     static class Resolver extends IndexNameExpressionResolver {
+        Resolver() {
+            super(new ThreadContext(Settings.EMPTY));
+        }
+
         @Override
         public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();

server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java

@@ -51,6 +51,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexingPressure;
@@ -147,7 +148,7 @@ public class TransportBulkActionIngestTests extends ESTestCase {
                 null, null, new ActionFilters(Collections.emptySet()), null,
                 new AutoCreateIndex(
                     SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                    new IndexNameExpressionResolver(),
+                    new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)),
                     new SystemIndices(emptyMap())
                 ), new IndexingPressure(SETTINGS), new SystemIndices(emptyMap())
             );

server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java

@@ -39,6 +39,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.rest.action.document.RestBulkAction;
@@ -215,6 +216,10 @@ public class TransportBulkActionTookTests extends ESTestCase {
     }
 
     static class Resolver extends IndexNameExpressionResolver {
+        Resolver() {
+            super(new ThreadContext(Settings.EMPTY));
+        }
+
         @Override
         public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();

server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java

@@ -37,6 +37,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -222,6 +223,10 @@ public class TransportMultiGetActionTests extends ESTestCase {
 
     static class Resolver extends IndexNameExpressionResolver {
 
+        Resolver() {
+            super(new ThreadContext(Settings.EMPTY));
+        }
+
         @Override
         public Index concreteSingleIndex(ClusterState state, IndicesRequest request) {
             return new Index("index1", randomBase64UUID());

server/src/test/java/org/elasticsearch/action/search/MultiSearchActionTookTests.java

@@ -32,6 +32,7 @@ import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
@@ -191,6 +192,10 @@ public class MultiSearchActionTookTests extends ESTestCase {
     }
 
     static class Resolver extends IndexNameExpressionResolver {
+        Resolver() {
+            super(new ThreadContext(Settings.EMPTY));
+        }
+
         @Override
         public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();

server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java

@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.SystemIndices;
@@ -194,7 +195,8 @@ public class AutoCreateIndexTests extends ESTestCase {
 
         ClusterSettings clusterSettings = new ClusterSettings(settings,
                 ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        AutoCreateIndex  autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, new IndexNameExpressionResolver(),
+        AutoCreateIndex autoCreateIndex = new AutoCreateIndex(settings, clusterSettings,
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)),
             new SystemIndices(org.elasticsearch.common.collect.Map.of()));
         assertThat(autoCreateIndex.getAutoCreate().isAutoCreateIndex(), equalTo(value));
 
@@ -222,7 +224,7 @@ public class AutoCreateIndexTests extends ESTestCase {
         SystemIndices systemIndices = new SystemIndices(org.elasticsearch.common.collect.Map.of("plugin",
             org.elasticsearch.common.collect.List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, ""))));
         return new AutoCreateIndex(settings, new ClusterSettings(settings,
-                ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(), systemIndices);
+            ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), systemIndices);
     }
 
     private void expectNotMatch(ClusterState clusterState, AutoCreateIndex autoCreateIndex, String index) {

server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java

@@ -50,6 +50,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
@@ -177,6 +178,10 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
     }
 
     class MyResolver extends IndexNameExpressionResolver {
+        MyResolver() {
+            super(new ThreadContext(Settings.EMPTY));
+        }
+
         @Override
         public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();

server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java

@@ -42,7 +42,9 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.rest.RestStatus;
@@ -171,7 +173,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         Action(String actionName, TransportService transportService, ClusterService clusterService,
                ThreadPool threadPool) {
             super(actionName, transportService, clusterService, threadPool,
-                    new ActionFilters(new HashSet<>()), Request::new, new IndexNameExpressionResolver());
+                    new ActionFilters(new HashSet<>()), Request::new, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)));
         }
 
         @Override

server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java

@@ -42,6 +42,7 @@ import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -103,7 +104,7 @@ public class BroadcastReplicationTests extends ESTestCase {
         transportService.start();
         transportService.acceptIncomingRequests();
         broadcastReplicationAction = new TestBroadcastReplicationAction(clusterService, transportService,
-                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(), null);
+                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), null);
     }
 
     @Override

server/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java

@@ -39,7 +39,9 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
@@ -136,6 +138,10 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
     }
 
     class MyResolver extends IndexNameExpressionResolver {
+        MyResolver() {
+            super(new ThreadContext(Settings.EMPTY));
+        }
+
         @Override
         public String[] concreteIndexNames(ClusterState state, IndicesRequest request) {
             return request.indices();

server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java

@@ -38,6 +38,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -223,6 +224,10 @@ public class TransportMultiTermVectorsActionTests extends ESTestCase {
 
     static class Resolver extends IndexNameExpressionResolver {
 
+        Resolver() {
+            super(new ThreadContext(Settings.EMPTY));
+        }
+
         @Override
         public Index concreteSingleIndex(ClusterState state, IndicesRequest request) {
             return new Index("index1", randomBase64UUID());

server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java

@@ -51,6 +51,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
@@ -65,8 +66,23 @@ import java.util.function.Supplier;
 
 public class ClusterModuleTests extends ModuleTestCase {
     private ClusterInfoService clusterInfoService = EmptyClusterInfoService.INSTANCE;
-    private ClusterService clusterService = new ClusterService(Settings.EMPTY,
-        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
+    private ClusterService clusterService;
+    private ThreadContext threadContext;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadContext = new ThreadContext(Settings.EMPTY);
+        clusterService = new ClusterService(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+    }
+
     static class FakeAllocationDecider extends AllocationDecider {
         protected FakeAllocationDecider() {
         }
@@ -121,7 +137,7 @@ public class ClusterModuleTests extends ModuleTestCase {
                     public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
                         return Collections.singletonList(new EnableAllocationDecider(settings, clusterSettings));
                     }
-                }), clusterInfoService, null));
+                }), clusterInfoService, null, threadContext));
         assertEquals(e.getMessage(),
             "Cannot specify allocation decider [" + EnableAllocationDecider.class.getName() + "] twice");
     }
@@ -133,7 +149,7 @@ public class ClusterModuleTests extends ModuleTestCase {
                 public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
                     return Collections.singletonList(new FakeAllocationDecider());
                 }
-            }), clusterInfoService, null);
+            }), clusterInfoService, null, threadContext);
         assertTrue(module.deciderList.stream().anyMatch(d -> d.getClass().equals(FakeAllocationDecider.class)));
     }
 
@@ -145,7 +161,7 @@ public class ClusterModuleTests extends ModuleTestCase {
                     return Collections.singletonMap(name, supplier);
                 }
             }
-        ), clusterInfoService, null);
+        ), clusterInfoService, null, threadContext);
     }
 
     public void testRegisterShardsAllocator() {
@@ -163,7 +179,7 @@ public class ClusterModuleTests extends ModuleTestCase {
     public void testUnknownShardsAllocator() {
         Settings settings = Settings.builder().put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), "dne").build();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
-            new ClusterModule(settings, clusterService, Collections.emptyList(), clusterInfoService, null));
+            new ClusterModule(settings, clusterService, Collections.emptyList(), clusterInfoService, null, threadContext));
         assertEquals("Unknown ShardsAllocator [dne]", e.getMessage());
     }
 
@@ -231,14 +247,15 @@ public class ClusterModuleTests extends ModuleTestCase {
 
     public void testRejectsReservedExistingShardsAllocatorName() {
         final ClusterModule clusterModule = new ClusterModule(Settings.EMPTY, clusterService,
-            Collections.singletonList(existingShardsAllocatorPlugin(GatewayAllocator.ALLOCATOR_NAME)), clusterInfoService, null);
+            Collections.singletonList(existingShardsAllocatorPlugin(GatewayAllocator.ALLOCATOR_NAME)), clusterInfoService, null,
+            threadContext);
         expectThrows(IllegalArgumentException.class, () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator()));
     }
 
     public void testRejectsDuplicateExistingShardsAllocatorName() {
         final ClusterModule clusterModule = new ClusterModule(Settings.EMPTY, clusterService,
-            Arrays.asList(existingShardsAllocatorPlugin("duplicate"), existingShardsAllocatorPlugin("duplicate")), clusterInfoService,
-            null);
+            Arrays.asList(existingShardsAllocatorPlugin("duplicate"), existingShardsAllocatorPlugin("duplicate")), clusterInfoService, null,
+            threadContext);
         expectThrows(IllegalArgumentException.class, () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator()));
     }
 

server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java

@@ -47,6 +47,7 @@ import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
@@ -78,7 +79,8 @@ import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class ClusterStateHealthTests extends ESTestCase {
-    private final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
+    private final IndexNameExpressionResolver indexNameExpressionResolver =
+        new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
 
     private static ThreadPool threadPool;
 

server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java

@@ -43,7 +43,8 @@ public class DateMathExpressionResolverTests extends ESTestCase {
 
     private final DateMathExpressionResolver expressionResolver = new DateMathExpressionResolver();
     private final Context context = new Context(
-            ClusterState.builder(new ClusterName("_name")).build(), IndicesOptions.strictExpand()
+        ClusterState.builder(new ClusterName("_name")).build(), IndicesOptions.strictExpand(),
+        false
     );
 
     public void testNormal() throws Exception {
@@ -146,7 +147,7 @@ public class DateMathExpressionResolverTests extends ESTestCase {
             // rounding to today 00:00
             now = DateTime.now(UTC).withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0);
         }
-        Context context = new Context(this.context.getState(), this.context.getOptions(), now.getMillis());
+        Context context = new Context(this.context.getState(), this.context.getOptions(), now.getMillis(), false);
         List<String> results = expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{yyyy.MM.dd|" + timeZone.getID() + "}}>"));
         assertThat(results.size(), equalTo(1));
         logger.info("timezone: [{}], now [{}], name: [{}]", timeZone, now, results.get(0));

server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionTests.java

@@ -36,12 +36,12 @@ public class IndexAbstractionTests extends ESTestCase {
         final String hiddenAliasName = "hidden_alias";
         AliasMetadata hiddenAliasMetadata = new AliasMetadata.Builder(hiddenAliasName).isHidden(true).build();
 
-        IndexMetadata hidden1 = buildIndexWithAlias("hidden1", hiddenAliasName, true);
-        IndexMetadata hidden2 = buildIndexWithAlias("hidden2", hiddenAliasName, true);
-        IndexMetadata hidden3 = buildIndexWithAlias("hidden3", hiddenAliasName, true);
+        IndexMetadata hidden1 = buildIndexWithAlias("hidden1", hiddenAliasName, true, Version.CURRENT, false);
+        IndexMetadata hidden2 = buildIndexWithAlias("hidden2", hiddenAliasName, true, Version.CURRENT, false);
+        IndexMetadata hidden3 = buildIndexWithAlias("hidden3", hiddenAliasName, true, Version.CURRENT, false);
 
-        IndexMetadata indexWithNonHiddenAlias = buildIndexWithAlias("nonhidden1", hiddenAliasName, false);
-        IndexMetadata indexWithUnspecifiedAlias = buildIndexWithAlias("nonhidden2", hiddenAliasName, null);
+        IndexMetadata indexWithNonHiddenAlias = buildIndexWithAlias("nonhidden1", hiddenAliasName, false, Version.CURRENT, false);
+        IndexMetadata indexWithUnspecifiedAlias = buildIndexWithAlias("nonhidden2", hiddenAliasName, null, Version.CURRENT, false);
 
         {
             IndexAbstraction.Alias allHidden = new IndexAbstraction.Alias(hiddenAliasMetadata, hidden1);
@@ -116,13 +116,15 @@ public class IndexAbstractionTests extends ESTestCase {
         }
     }
 
-    private IndexMetadata buildIndexWithAlias(String indexName, String aliasName, @Nullable Boolean aliasIsHidden) {
+    private IndexMetadata buildIndexWithAlias(String indexName, String aliasName, @Nullable Boolean aliasIsHidden,
+                                              Version indexCreationVersion, boolean isSystem) {
         final AliasMetadata.Builder aliasMetadata = new AliasMetadata.Builder(aliasName);
         if (Objects.nonNull(aliasIsHidden) || randomBoolean()) {
             aliasMetadata.isHidden(aliasIsHidden);
         }
         return new IndexMetadata.Builder(indexName)
-            .settings(settings(Version.CURRENT))
+            .settings(settings(indexCreationVersion))
+            .system(isSystem)
             .numberOfShards(1)
             .numberOfReplicas(0)
             .putAlias(aliasMetadata)

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverAliasIterationTests.java

@@ -19,10 +19,13 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
 public class IndexNameExpressionResolverAliasIterationTests extends IndexNameExpressionResolverTests {
 
-    protected IndexNameExpressionResolver getIndexNameExpressionResolver() {
-        return new IndexNameExpressionResolver() {
+    protected IndexNameExpressionResolver createIndexNameExpressionResolver() {
+        return new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) {
             @Override
             boolean iterateIndexAliases(int indexAliasesSize, int resolvedExpressionsSize) {
                 return true;

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverExpressionsIterationTests.java

@@ -19,10 +19,13 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
 public class IndexNameExpressionResolverExpressionsIterationTests extends IndexNameExpressionResolverTests {
 
-    protected IndexNameExpressionResolver getIndexNameExpressionResolver() {
-        return new IndexNameExpressionResolver() {
+    protected IndexNameExpressionResolver createIndexNameExpressionResolver() {
+        return new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) {
             @Override
             boolean iterateIndexAliases(int indexAliasesSize, int resolvedExpressionsSize) {
                 return false;

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java

@@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.ClusterName;
@@ -33,6 +34,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata.State;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
@@ -48,10 +50,12 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.DataStreamTestHelper.createBackingIndex;
 import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_HIDDEN_SETTING;
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
@@ -67,15 +71,21 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public class IndexNameExpressionResolverTests extends ESTestCase {
     private IndexNameExpressionResolver indexNameExpressionResolver;
+    private ThreadContext threadContext;
 
-    protected IndexNameExpressionResolver getIndexNameExpressionResolver() {
-        return new IndexNameExpressionResolver();
+    private ThreadContext createThreadContext() {
+        return new ThreadContext(Settings.EMPTY);
+    }
+
+    protected IndexNameExpressionResolver createIndexNameExpressionResolver(ThreadContext threadContext) {
+        return new IndexNameExpressionResolver(threadContext);
     }
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        indexNameExpressionResolver = getIndexNameExpressionResolver();
+        threadContext = createThreadContext();
+        indexNameExpressionResolver = createIndexNameExpressionResolver(threadContext);
     }
 
     public void testIndexOptionsStrict() {
@@ -89,7 +99,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         IndicesOptions[] indicesOptions = new IndicesOptions[]{ IndicesOptions.strictExpandOpen(), IndicesOptions.strictExpand()};
         for (IndicesOptions options : indicesOptions) {
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options, false);
             String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
@@ -138,26 +148,27 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             assertEquals("foo", results[0]);
         }
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen(), false);
         String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
 
         results = indexNameExpressionResolver.concreteIndexNames(context, (String[])null);
         assertEquals(3, results.length);
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpand());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpand(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(4, results.length);
 
         results = indexNameExpressionResolver.concreteIndexNames(context, (String[])null);
         assertEquals(4, results.length);
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo*");
         assertEquals(3, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo"));
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpand());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpand(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo*");
         assertEquals(4, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo", "foofoo-closed"));
@@ -173,7 +184,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         IndicesOptions[] indicesOptions = new IndicesOptions[]{IndicesOptions.lenientExpandOpen(), IndicesOptions.lenientExpand()};
         for (IndicesOptions options : indicesOptions) {
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options, false);
             String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
@@ -210,20 +221,21 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             assertEquals("foo", results[0]);
         }
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpand());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpand(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(Arrays.toString(results), 4, results.length);
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context,  "foofoo*");
         assertEquals(3, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo"));
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpand());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpand(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context, "foofoo*");
         assertEquals(4, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foobar", "foofoo", "foofoo-closed"));
@@ -242,7 +254,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         IndicesOptions[] indicesOptions = new IndicesOptions[]{expandOpen, expand};
 
         for (IndicesOptions options : indicesOptions) {
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options, false);
             String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo");
             assertEquals(1, results.length);
             assertEquals("foo", results[0]);
@@ -264,11 +276,11 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             }
         }
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, expandOpen);
+        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, expandOpen, false);
         String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
 
-        context = new IndexNameExpressionResolver.Context(state, expand);
+        context = new IndexNameExpressionResolver.Context(state, expand, false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(4, results.length);
     }
@@ -286,7 +298,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         // Only closed
         IndicesOptions options = IndicesOptions.fromOptions(false, true, false, true, false);
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
+        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options, false);
         String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(1, results.length);
         assertEquals("foo", results[0]);
@@ -311,7 +323,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         // Only open
         options = IndicesOptions.fromOptions(false, true, true, false, false);
-        context = new IndexNameExpressionResolver.Context(state, options);
+        context = new IndexNameExpressionResolver.Context(state, options, false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(2, results.length);
         assertThat(results, arrayContainingInAnyOrder("bar", "foobar"));
@@ -335,7 +347,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         // Open and closed
         options = IndicesOptions.fromOptions(false, true, true, true, false);
-        context = new IndexNameExpressionResolver.Context(state, options);
+        context = new IndexNameExpressionResolver.Context(state, options, false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
         assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "foo"));
@@ -374,7 +386,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         // open closed and hidden
         options = IndicesOptions.fromOptions(false, true, true, true, true);
-        context = new IndexNameExpressionResolver.Context(state, options);
+        context = new IndexNameExpressionResolver.Context(state, options, false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(7, results.length);
         assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "foo", "hidden", "hidden-closed", ".hidden", ".hidden-closed"));
@@ -416,7 +428,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         // open and hidden
         options = IndicesOptions.fromOptions(false, true, true, false, true);
-        context = new IndexNameExpressionResolver.Context(state, options);
+        context = new IndexNameExpressionResolver.Context(state, options, false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(4, results.length);
         assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "hidden", ".hidden"));
@@ -435,7 +447,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         // closed and hidden
         options = IndicesOptions.fromOptions(false, true, false, true, true);
-        context = new IndexNameExpressionResolver.Context(state, options);
+        context = new IndexNameExpressionResolver.Context(state, options, false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "hidden-closed", ".hidden-closed"));
@@ -454,7 +466,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         // only hidden
         options = IndicesOptions.fromOptions(false, true, false, false, true);
-        context = new IndexNameExpressionResolver.Context(state, options);
+        context = new IndexNameExpressionResolver.Context(state, options, false);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertThat(results, emptyArray());
 
@@ -468,7 +480,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         assertThat(results, arrayContainingInAnyOrder("hidden-closed"));
 
         options = IndicesOptions.fromOptions(false, false, true, true, true);
-        IndexNameExpressionResolver.Context context2 = new IndexNameExpressionResolver.Context(state, options);
+        IndexNameExpressionResolver.Context context2 = new IndexNameExpressionResolver.Context(state, options, false);
         IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                 () -> indexNameExpressionResolver.concreteIndexNames(context2, "-*"));
         assertThat(infe.getResourceId().toString(), equalTo("[-*]"));
@@ -485,7 +497,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         //ignore unavailable and allow no indices
         {
             IndicesOptions noExpandLenient = IndicesOptions.fromOptions(true, true, false, false, randomBoolean());
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandLenient);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandLenient, false);
             String[] results = indexNameExpressionResolver.concreteIndexNames(context, "baz*");
             assertThat(results, emptyArray());
 
@@ -507,7 +519,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         //ignore unavailable but don't allow no indices
         {
             IndicesOptions noExpandDisallowEmpty = IndicesOptions.fromOptions(true, false, false, false, randomBoolean());
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandDisallowEmpty);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandDisallowEmpty, false);
 
             {
                 IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
@@ -532,7 +544,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         //error on unavailable but allow no indices
         {
             IndicesOptions noExpandErrorUnavailable = IndicesOptions.fromOptions(false, true, false, false, randomBoolean());
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandErrorUnavailable);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandErrorUnavailable, false);
             {
                 String[] results = indexNameExpressionResolver.concreteIndexNames(context, "baz*");
                 assertThat(results, emptyArray());
@@ -558,7 +570,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         //error on both unavailable and no indices
         {
             IndicesOptions noExpandStrict = IndicesOptions.fromOptions(false, false, false, false, randomBoolean());
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandStrict);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandStrict, false);
             IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                     () -> indexNameExpressionResolver.concreteIndexNames(context, "baz*"));
             assertThat(infe.getIndex().getName(), equalTo("baz*"));
@@ -585,7 +597,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         {
             IndexNameExpressionResolver.Context context =
-                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
+                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed(), false);
             IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                     () -> indexNameExpressionResolver.concreteIndexNames(context, "baz*"));
             assertThat(infe.getIndex().getName(), equalTo("baz*"));
@@ -593,7 +605,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         {
             IndexNameExpressionResolver.Context context =
-                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
+                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed(), false);
             IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                     () -> indexNameExpressionResolver.concreteIndexNames(context, "foo", "baz*"));
             assertThat(infe.getIndex().getName(), equalTo("baz*"));
@@ -601,7 +613,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         {
             IndexNameExpressionResolver.Context context =
-                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
+                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed(), false);
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                     () -> indexNameExpressionResolver.concreteIndexNames(context, "foofoobar"));
             assertThat(e.getMessage(), containsString("alias [foofoobar] has more than one index associated with it"));
@@ -609,7 +621,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         {
             IndexNameExpressionResolver.Context context =
-                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
+                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed(), false);
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                     () -> indexNameExpressionResolver.concreteIndexNames(context, "foo", "foofoobar"));
             assertThat(e.getMessage(), containsString("alias [foofoobar] has more than one index associated with it"));
@@ -617,7 +629,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         {
             IndexNameExpressionResolver.Context context =
-                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
+                new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed(), false);
             IndexClosedException ince = expectThrows(IndexClosedException.class,
                     () -> indexNameExpressionResolver.concreteIndexNames(context, "foofoo-closed", "foofoobar"));
             assertThat(ince.getMessage(), equalTo("closed"));
@@ -625,7 +637,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         }
 
         IndexNameExpressionResolver.Context context =
-            new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed());
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.strictSingleIndexNoExpandForbidClosed(), false);
         String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo", "barbaz");
         assertEquals(2, results.length);
         assertThat(results, arrayContainingInAnyOrder("foo", "foofoo"));
@@ -635,7 +647,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(Metadata.builder().build()).build();
 
         IndicesOptions options = IndicesOptions.strictExpandOpen();
-        final IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
+        final IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options, false);
         String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertThat(results, emptyArray());
 
@@ -656,7 +668,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
 
         final IndexNameExpressionResolver.Context context2 =
-            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context2, Strings.EMPTY_ARRAY);
         assertThat(results, emptyArray());
         results = indexNameExpressionResolver.concreteIndexNames(context2, "foo");
@@ -667,7 +679,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         assertThat(results, emptyArray());
 
         final IndexNameExpressionResolver.Context context3 =
-            new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, false, true, false));
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, false, true, false), false);
         IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                 () -> indexNameExpressionResolver.concreteIndexNames(context3, Strings.EMPTY_ARRAY));
         assertThat(infe.getResourceId().toString(), equalTo("[_all]"));
@@ -692,7 +704,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                 .put(indexBuilder("testXXX"))
                 .put(indexBuilder("kuku"));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen(), false);
 
         IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                 () -> indexNameExpressionResolver.concreteIndexNames(context, "testZZZ"));
@@ -704,7 +717,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                 .put(indexBuilder("testXXX"))
                 .put(indexBuilder("kuku"));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
 
         assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testXXX", "testZZZ")),
             equalTo(newHashSet("testXXX")));
@@ -715,7 +729,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                 .put(indexBuilder("testXXX"))
                 .put(indexBuilder("kuku"));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen(), false);
 
         IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                 () -> indexNameExpressionResolver.concreteIndexNames(context, "testMo", "testMahdy"));
@@ -727,7 +742,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                 .put(indexBuilder("testXXX"))
                 .put(indexBuilder("kuku"));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, new String[]{})),
             equalTo(newHashSet("kuku", "testXXX")));
     }
@@ -735,7 +751,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         Metadata.Builder mdBuilder = Metadata.builder();
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state,
-            IndicesOptions.fromOptions(false, false, true, true));
+            IndicesOptions.fromOptions(false, false, true, true), false);
         IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                 () -> indexNameExpressionResolver.concreteIndices(context, new String[]{}));
         assertThat(infe.getMessage(), is("no such index [null] and no indices exist"));
@@ -745,7 +761,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         Metadata.Builder mdBuilder = Metadata.builder();
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state,
-            IndicesOptions.fromOptions(false, false, false, false));
+            IndicesOptions.fromOptions(false, false, false, false), false);
         IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                 () -> indexNameExpressionResolver.concreteIndices(context, new String[]{}));
         assertThat(infe.getMessage(), is("no such index [_all] and no indices exist"));
@@ -761,16 +777,16 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
         IndexNameExpressionResolver.Context context =
-            new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, false));
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, false), false);
         assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")),
             equalTo(new HashSet<String>()));
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, false));
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, false), false);
         assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")),
             equalTo(newHashSet("testXXX", "testXXY")));
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, true));
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, true), false);
         assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")),
             equalTo(newHashSet("testXYY")));
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, true));
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, true), false);
         assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")),
             equalTo(newHashSet("testXXX", "testXXY", "testXYY")));
     }
@@ -788,7 +804,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state,
-                IndicesOptions.fromOptions(true, true, true, true));
+                IndicesOptions.fromOptions(true, true, true, true), false);
         assertThat(newHashSet(indexNameExpressionResolver.concreteIndexNames(context, "testX*")),
                 equalTo(newHashSet("testXXX", "testXXY", "testXYY")));
 
@@ -1076,7 +1092,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
             {
                 ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(Metadata.builder().build()).build();
-                IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, indicesOptions);
+                IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, indicesOptions, false);
 
                 // with no indices, asking for all indices should return empty list or exception, depending on indices options
                 if (indicesOptions.allowNoIndices()) {
@@ -1095,7 +1111,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                         .put(indexBuilder("bbb").state(State.OPEN).putAlias(AliasMetadata.builder("bbb_alias1")))
                         .put(indexBuilder("ccc").state(State.CLOSE).putAlias(AliasMetadata.builder("ccc_alias1")));
                 ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
-                IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, indicesOptions);
+                IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, indicesOptions, false);
                 if (indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed() || indicesOptions.allowNoIndices()) {
                     String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(context, allIndices);
                     assertThat(concreteIndices, notNullValue());
@@ -1125,7 +1141,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
                     .put(indexBuilder("bbb").state(State.OPEN).putAlias(AliasMetadata.builder("bbb_alias1")))
                     .put(indexBuilder("ccc").state(State.CLOSE).putAlias(AliasMetadata.builder("ccc_alias1")));
             ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, indicesOptions);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, indicesOptions, false);
 
             // asking for non existing wildcard pattern should return empty list or exception
             if (indicesOptions.allowNoIndices()) {
@@ -1254,20 +1270,20 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
         IndexNameExpressionResolver.Context contextICE =
-            new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpenAndForbidClosed());
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpenAndForbidClosed(), false);
         expectThrows(IndexClosedException.class, () -> indexNameExpressionResolver.concreteIndexNames(contextICE, "foo1-closed"));
         expectThrows(IndexClosedException.class, () -> indexNameExpressionResolver.concreteIndexNames(contextICE, "foobar1-closed"));
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true,
                 contextICE.getOptions().allowNoIndices(), contextICE.getOptions().expandWildcardsOpen(),
-            contextICE.getOptions().expandWildcardsClosed(), contextICE.getOptions()));
+            contextICE.getOptions().expandWildcardsClosed(), contextICE.getOptions()), false);
         String[] results = indexNameExpressionResolver.concreteIndexNames(context, "foo1-closed");
         assertThat(results, emptyArray());
 
         results = indexNameExpressionResolver.concreteIndexNames(context, "foobar1-closed");
         assertThat(results, emptyArray());
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context, "foo1-closed");
         assertThat(results, arrayWithSize(1));
         assertThat(results, arrayContaining("foo1-closed"));
@@ -1277,7 +1293,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         assertThat(results, arrayContaining("foo1-closed"));
 
         // testing an alias pointing to three indices:
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpenAndForbidClosed());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpenAndForbidClosed(), false);
         try {
             indexNameExpressionResolver.concreteIndexNames(context, "foobar2-closed");
             fail("foo2-closed should be closed, but it is open");
@@ -1287,12 +1303,12 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
 
         context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true,
             context.getOptions().allowNoIndices(), context.getOptions().expandWildcardsOpen(),
-            context.getOptions().expandWildcardsClosed(), context.getOptions()));
+            context.getOptions().expandWildcardsClosed(), context.getOptions()), false);
         results = indexNameExpressionResolver.concreteIndexNames(context, "foobar2-closed");
         assertThat(results, arrayWithSize(1));
         assertThat(results, arrayContaining("foo3"));
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         results = indexNameExpressionResolver.concreteIndexNames(context, "foobar2-closed");
         assertThat(results, arrayWithSize(3));
         assertThat(results, arrayContainingInAnyOrder("foo1-closed", "foo2-closed", "foo3"));
@@ -1305,7 +1321,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         IndicesOptions[] indicesOptions = new IndicesOptions[]{ IndicesOptions.strictExpandOpen(), IndicesOptions.strictExpand(),
                 IndicesOptions.lenientExpandOpen(), IndicesOptions.strictExpandOpenAndForbidClosed()};
         for (IndicesOptions options : indicesOptions) {
-            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options, false);
             String[] results = indexNameExpressionResolver.concreteIndexNames(context, "index1", "index1", "alias1");
             assertThat(results, equalTo(new String[]{"index1"}));
         }
@@ -1325,11 +1341,12 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             .put(indexBuilder("test-1").state(IndexMetadata.State.CLOSE).putAlias(AliasMetadata.builder("alias-1")));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         String[] strings = indexNameExpressionResolver.concreteIndexNames(context, "alias-*");
         assertArrayEquals(new String[] {"test-0"}, strings);
 
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen(), false);
         strings = indexNameExpressionResolver.concreteIndexNames(context, "alias-*");
 
         assertArrayEquals(new String[] {"test-0"}, strings);
@@ -1740,7 +1757,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
     public void testInvalidIndex() {
         Metadata.Builder mdBuilder = Metadata.builder().put(indexBuilder("test"));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
 
         InvalidIndexNameException iine = expectThrows(InvalidIndexNameException.class,
             () -> indexNameExpressionResolver.concreteIndexNames(context, "_foo"));
@@ -1811,6 +1829,86 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         }
     }
 
+    public void testFullWildcardSystemIndexResolutionAllowed() {
+        ClusterState state = systemIndexTestClusterState();
+        SearchRequest request = new SearchRequest(randomFrom("*", "_all"));
+
+        List<String> indexNames = resolveConcreteIndexNameList(state, request);
+        assertThat(indexNames, containsInAnyOrder("some-other-index", ".ml-stuff", ".ml-meta", ".watches"));
+    }
+
+    public void testWildcardSystemIndexResolutionMultipleMatchesAllowed() {
+        ClusterState state = systemIndexTestClusterState();
+        SearchRequest request = new SearchRequest(".w*");
+
+        List<String> indexNames = resolveConcreteIndexNameList(state, request);
+        assertThat(indexNames, containsInAnyOrder(".watches"));
+    }
+
+    public void testWildcardSystemIndexResolutionSingleMatchAllowed() {
+        ClusterState state = systemIndexTestClusterState();
+        SearchRequest request = new SearchRequest(".ml-*");
+
+        List<String> indexNames = resolveConcreteIndexNameList(state, request);
+        assertThat(indexNames, containsInAnyOrder(".ml-meta", ".ml-stuff"));
+    }
+
+    public void testSingleSystemIndexResolutionAllowed() {
+        ClusterState state = systemIndexTestClusterState();
+        SearchRequest request = new SearchRequest(".ml-meta");
+
+        List<String> indexNames = resolveConcreteIndexNameList(state, request);
+        assertThat(indexNames, containsInAnyOrder(".ml-meta"));
+    }
+
+    public void testFullWildcardSystemIndexResolutionDeprecated() {
+        threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.FALSE.toString());
+        ClusterState state = systemIndexTestClusterState();
+        SearchRequest request = new SearchRequest(randomFrom("*", "_all"));
+
+        List<String> indexNames = resolveConcreteIndexNameList(state, request);
+        assertThat(indexNames, containsInAnyOrder("some-other-index", ".ml-stuff", ".ml-meta", ".watches"));
+        assertWarnings("this request accesses system indices: [.ml-meta, .ml-stuff, .watches], but in a future major version, " +
+            "direct access to system indices will be prevented by default");
+
+    }
+
+    public void testSingleSystemIndexResolutionDeprecated() {
+        threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.FALSE.toString());
+        ClusterState state = systemIndexTestClusterState();
+        SearchRequest request = new SearchRequest(".ml-meta");
+
+        List<String> indexNames = resolveConcreteIndexNameList(state, request);
+        assertThat(indexNames, containsInAnyOrder(".ml-meta"));
+        assertWarnings("this request accesses system indices: [.ml-meta], but in a future major version, direct access " +
+            "to system indices will be prevented by default");
+
+    }
+
+    public void testWildcardSystemIndexReslutionSingleMatchDeprecated() {
+        threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.FALSE.toString());
+        ClusterState state = systemIndexTestClusterState();
+        SearchRequest request = new SearchRequest(".w*");
+
+        List<String> indexNames = resolveConcreteIndexNameList(state, request);
+        assertThat(indexNames, containsInAnyOrder(".watches"));
+        assertWarnings("this request accesses system indices: [.watches], but in a future major version, direct access " +
+            "to system indices will be prevented by default");
+
+    }
+
+    public void testWildcardSystemIndexResolutionMultipleMatchesDeprecated() {
+        threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, Boolean.FALSE.toString());
+        ClusterState state = systemIndexTestClusterState();
+        SearchRequest request = new SearchRequest(".ml-*");
+
+        List<String> indexNames = resolveConcreteIndexNameList(state, request);
+        assertThat(indexNames, containsInAnyOrder(".ml-meta", ".ml-stuff"));
+        assertWarnings("this request accesses system indices: [.ml-meta, .ml-stuff], but in a future major version, direct access " +
+            "to system indices will be prevented by default");
+
+    }
+
     public void testDataStreams() {
         final String dataStreamName = "my-data-stream";
         IndexMetadata index1 = createBackingIndex(dataStreamName, 1).build();
@@ -2049,4 +2147,21 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         names = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.lenientExpand(), "*", "-*");
         assertThat(names, empty());
     }
+
+    private ClusterState systemIndexTestClusterState() {
+        Settings settings = Settings.builder().build();
+        Metadata.Builder mdBuilder = Metadata.builder()
+            .put(indexBuilder(".ml-meta", settings).state(State.OPEN).system(true))
+            .put(indexBuilder(".watches", settings).state(State.OPEN).system(true))
+            .put(indexBuilder(".ml-stuff", settings).state(State.OPEN).system(true))
+            .put(indexBuilder("some-other-index").state(State.OPEN));
+        return ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+    }
+
+    private List<String> resolveConcreteIndexNameList(ClusterState state, SearchRequest request) {
+        return Arrays
+            .stream(indexNameExpressionResolver.concreteIndices(state, request))
+            .map(i -> i.getName())
+            .collect(Collectors.toList());
+    }
 }

server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java

@@ -48,7 +48,8 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
         IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         assertThat(newHashSet(resolver.resolve(context, Collections.singletonList("testXXX"))), equalTo(newHashSet("testXXX")));
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testXXX", "testYYY"))), equalTo(newHashSet("testXXX", "testYYY")));
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testXXX", "ku*"))), equalTo(newHashSet("testXXX", "kuku")));
@@ -76,7 +77,8 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
         IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         assertThat(newHashSet(resolver.resolve(context, Arrays.asList("testYY*", "alias*"))),
             equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
         assertThat(newHashSet(resolver.resolve(context, Collections.singletonList("-kuku"))), equalTo(newHashSet("-kuku")));
@@ -99,12 +101,12 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
 
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state,
-            IndicesOptions.fromOptions(true, true, true, true));
+            IndicesOptions.fromOptions(true, true, true, true), false);
         assertThat(newHashSet(resolver.resolve(context, Collections.singletonList("testX*"))),
             equalTo(newHashSet("testXXX", "testXXY", "testXYY")));
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, true));
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, true), false);
         assertThat(newHashSet(resolver.resolve(context, Collections.singletonList("testX*"))), equalTo(newHashSet("testXYY")));
-        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, false));
+        context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, true, false), false);
         assertThat(newHashSet(resolver.resolve(context, Collections.singletonList("testX*"))), equalTo(newHashSet("testXXX", "testXXY")));
     }
 
@@ -121,7 +123,8 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
         IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         assertThat(newHashSet(resolver.resolve(context, Collections.singletonList("test*X*"))),
             equalTo(newHashSet("testXXX", "testXXY", "testXYY")));
         assertThat(newHashSet(resolver.resolve(context, Collections.singletonList("test*X*Y"))), equalTo(newHashSet("testXXY", "testXYY")));
@@ -140,7 +143,8 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
         IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
 
-        IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
+        IndexNameExpressionResolver.Context context =
+            new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen(), false);
         assertThat(newHashSet(resolver.resolve(context, Collections.singletonList("_all"))),
             equalTo(newHashSet("testXXX", "testXYY", "testYYY")));
     }
@@ -158,15 +162,15 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, false,
             false, false);
         IndexNameExpressionResolver.Context indicesAndAliasesContext =
-            new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions);
+            new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions, false);
         // ignoreAliases option is set, WildcardExpressionResolver throws error when
         IndicesOptions skipAliasesIndicesOptions = IndicesOptions.fromOptions(true, true, true, false, true, false, true, false);
         IndexNameExpressionResolver.Context skipAliasesLenientContext =
-            new IndexNameExpressionResolver.Context(state, skipAliasesIndicesOptions);
+            new IndexNameExpressionResolver.Context(state, skipAliasesIndicesOptions, false);
         // ignoreAliases option is set, WildcardExpressionResolver resolves the provided expressions only against the defined indices
         IndicesOptions errorOnAliasIndicesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
         IndexNameExpressionResolver.Context skipAliasesStrictContext =
-            new IndexNameExpressionResolver.Context(state, errorOnAliasIndicesOptions);
+            new IndexNameExpressionResolver.Context(state, errorOnAliasIndicesOptions, false);
 
         {
             List<String> indices = resolver.resolve(indicesAndAliasesContext, Collections.singletonList("foo_a*"));
@@ -232,7 +236,7 @@ public class WildcardExpressionResolverTests extends ESTestCase {
             IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, false,
                 false, false);
             IndexNameExpressionResolver.Context indicesAndAliasesContext =
-                new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions);
+                new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions, false);
 
             // data streams are not included but expression matches the data stream
             List<String> indices = resolver.resolve(indicesAndAliasesContext, Collections.singletonList("foo_*"));
@@ -247,7 +251,7 @@ public class WildcardExpressionResolverTests extends ESTestCase {
             IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, false,
                 false, false);
             IndexNameExpressionResolver.Context indicesAliasesAndDataStreamsContext = new IndexNameExpressionResolver.Context(state,
-                indicesAndAliasesOptions, false, false, true);
+                indicesAndAliasesOptions, false, false, true, false);
 
             // data stream's corresponding backing indices are resolved
             List<String> indices = resolver.resolve(indicesAliasesAndDataStreamsContext, Collections.singletonList("foo_*"));
@@ -264,7 +268,7 @@ public class WildcardExpressionResolverTests extends ESTestCase {
             IndicesOptions indicesAliasesAndExpandHiddenOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false,
                 true, true, false, false, false);
             IndexNameExpressionResolver.Context indicesAliasesDataStreamsAndHiddenIndices = new IndexNameExpressionResolver.Context(state,
-                indicesAliasesAndExpandHiddenOptions, false, false, true);
+                indicesAliasesAndExpandHiddenOptions, false, false, true, false);
 
             // data stream's corresponding backing indices are resolved
             List<String> indices = resolver.resolve(indicesAliasesDataStreamsAndHiddenIndices, Collections.singletonList("foo_*"));
@@ -290,12 +294,12 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         // expressions against the defined indices and aliases
         IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, false, false);
         IndexNameExpressionResolver.Context indicesAndAliasesContext =
-            new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions);
+            new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions, false);
 
         // ignoreAliases option is set, WildcardExpressionResolver resolves the provided expressions
         // only against the defined indices
         IndicesOptions onlyIndicesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
-        IndexNameExpressionResolver.Context onlyIndicesContext = new IndexNameExpressionResolver.Context(state, onlyIndicesOptions);
+        IndexNameExpressionResolver.Context onlyIndicesContext = new IndexNameExpressionResolver.Context(state, onlyIndicesOptions, false);
 
         {
             Set<String> matches = IndexNameExpressionResolver.WildcardExpressionResolver.matches(indicesAndAliasesContext,

server/src/test/java/org/elasticsearch/index/IndexModuleTests.java

@@ -48,6 +48,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
@@ -179,7 +180,7 @@ public class IndexModuleTests extends ESTestCase {
                 engineFactory,
                 Collections.emptyMap(),
                 () -> true,
-                new IndexNameExpressionResolver(),
+                new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)),
                 Collections.emptyMap());
         module.setReaderWrapper(s -> new Wrapper());
 
@@ -201,7 +202,7 @@ public class IndexModuleTests extends ESTestCase {
         final Map<String, IndexStorePlugin.DirectoryFactory> indexStoreFactories = singletonMap(
             "foo_store", new FooFunction());
         final IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), indexStoreFactories,
-            () -> true, new IndexNameExpressionResolver(), Collections.emptyMap());
+            () -> true, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), Collections.emptyMap());
 
         final IndexService indexService = newIndexService(module);
         assertThat(indexService.getDirectoryFactory(), instanceOf(FooFunction.class));
@@ -514,7 +515,7 @@ public class IndexModuleTests extends ESTestCase {
             new InternalEngineFactory(),
             Collections.emptyMap(),
             () -> true,
-            new IndexNameExpressionResolver(),
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)),
             recoveryStateFactories);
 
         final IndexService indexService = newIndexService(module);
@@ -535,7 +536,7 @@ public class IndexModuleTests extends ESTestCase {
 
     private static IndexModule createIndexModule(IndexSettings indexSettings, AnalysisRegistry emptyAnalysisRegistry) {
         return new IndexModule(indexSettings, emptyAnalysisRegistry, new InternalEngineFactory(), Collections.emptyMap(), () -> true,
-            new IndexNameExpressionResolver(), Collections.emptyMap());
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), Collections.emptyMap());
     }
 
     class CustomQueryCache implements QueryCache {

server/src/test/java/org/elasticsearch/index/query/SearchIndexNameMatcherTests.java

@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -49,8 +50,10 @@ public class SearchIndexNameMatcherTests extends ESTestCase {
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(state);
 
-        matcher = new SearchIndexNameMatcher("index1", "", clusterService, new IndexNameExpressionResolver());
-        remoteMatcher = new SearchIndexNameMatcher("index1", "cluster", clusterService, new IndexNameExpressionResolver());
+        matcher = new SearchIndexNameMatcher("index1", "", clusterService,
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)));
+        remoteMatcher = new SearchIndexNameMatcher("index1", "cluster", clusterService,
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)));
     }
 
     private static IndexMetadata.Builder indexBuilder(String index) {

server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java

@@ -81,6 +81,7 @@ import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
@@ -153,7 +154,7 @@ public class ClusterStateChanges {
         shardStartedClusterStateTaskExecutor
             = new ShardStateAction.ShardStartedClusterStateTaskExecutor(allocationService, null, () -> Priority.NORMAL, logger);
         ActionFilters actionFilters = new ActionFilters(Collections.emptySet());
-        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
         DestructiveOperations destructiveOperations = new DestructiveOperations(SETTINGS, clusterSettings);
         Environment environment = TestEnvironment.newEnvironment(SETTINGS);
         Transport transport = mock(Transport.class); // it's not used

server/src/test/java/org/elasticsearch/rest/BaseRestHandlerTests.java

@@ -22,6 +22,7 @@ package org.elasticsearch.rest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -29,6 +30,8 @@ import org.elasticsearch.rest.action.cat.AbstractCatAction;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestChannel;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -39,9 +42,24 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.hamcrest.core.StringContains.containsString;
 import static org.hamcrest.object.HasToString.hasToString;
-import static org.mockito.Mockito.mock;
 
 public class BaseRestHandlerTests extends ESTestCase {
+    private NodeClient mockClient;
+    private ThreadPool threadPool;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(this.getClass().getSimpleName() + "ThreadPool");
+        mockClient = new NodeClient(Settings.EMPTY, threadPool);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        threadPool.shutdown();
+        mockClient.close();
+    }
 
     public void testOneUnconsumedParameters() throws Exception {
         final AtomicBoolean executed = new AtomicBoolean();
@@ -69,7 +87,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         final IllegalArgumentException e =
-            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
+            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mockClient));
         assertThat(e, hasToString(containsString("request [/] contains unrecognized parameter: [unconsumed]")));
         assertFalse(executed.get());
     }
@@ -101,7 +119,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         final IllegalArgumentException e =
-            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
+            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mockClient));
         assertThat(e, hasToString(containsString("request [/] contains unrecognized parameters: [unconsumed-first], [unconsumed-second]")));
         assertFalse(executed.get());
     }
@@ -145,7 +163,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         final IllegalArgumentException e =
-            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
+            expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mockClient));
         assertThat(
             e,
             hasToString(containsString(
@@ -188,7 +206,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         params.put("response_param", randomAlphaOfLength(8));
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
-        handler.handleRequest(request, channel, mock(NodeClient.class));
+        handler.handleRequest(request, channel, mockClient);
         assertTrue(executed.get());
     }
 
@@ -218,7 +236,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         params.put("human", null);
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
-        handler.handleRequest(request, channel, mock(NodeClient.class));
+        handler.handleRequest(request, channel, mockClient);
         assertTrue(executed.get());
     }
 
@@ -262,7 +280,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         params.put("time", randomAlphaOfLength(8));
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
-        handler.handleRequest(request, channel, mock(NodeClient.class));
+        handler.handleRequest(request, channel, mockClient);
         assertTrue(executed.get());
     }
 
@@ -291,7 +309,7 @@ public class BaseRestHandlerTests extends ESTestCase {
                     .withContent(new BytesArray(builder.toString()), XContentType.JSON)
                     .build();
             final RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
-            handler.handleRequest(request, channel, mock(NodeClient.class));
+            handler.handleRequest(request, channel, mockClient);
             assertTrue(executed.get());
         }
     }
@@ -317,7 +335,7 @@ public class BaseRestHandlerTests extends ESTestCase {
 
         final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).build();
         final RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
-        handler.handleRequest(request, channel, mock(NodeClient.class));
+        handler.handleRequest(request, channel, mockClient);
         assertTrue(executed.get());
     }
 
@@ -346,7 +364,7 @@ public class BaseRestHandlerTests extends ESTestCase {
                     .build();
             final RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
             final IllegalArgumentException e =
-                    expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
+                    expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mockClient));
             assertThat(e, hasToString(containsString("request [GET /] does not support having a body")));
             assertFalse(executed.get());
         }

server/src/test/java/org/elasticsearch/rest/RestControllerTests.java

@@ -33,6 +33,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.http.HttpInfo;
 import org.elasticsearch.http.HttpRequest;
 import org.elasticsearch.http.HttpResponse;
@@ -40,8 +41,10 @@ import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpNodeClient;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.usage.UsageService;
+import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -75,6 +78,7 @@ public class RestControllerTests extends ESTestCase {
     private RestController restController;
     private HierarchyCircuitBreakerService circuitBreakerService;
     private UsageService usageService;
+    private NodeClient client;
 
     @Before
     public void setup() {
@@ -91,7 +95,8 @@ public class RestControllerTests extends ESTestCase {
         inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
 
         HttpServerTransport httpServerTransport = new TestHttpServerTransport();
-        restController = new RestController(Collections.emptySet(), null, null, circuitBreakerService, usageService);
+        client = new NoOpNodeClient(this.getTestName());
+        restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/",
             (request, channel, client) -> channel.sendResponse(
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
@@ -102,8 +107,13 @@ public class RestControllerTests extends ESTestCase {
         httpServerTransport.start();
     }
 
+    @After
+    public void teardown() throws IOException {
+        IOUtils.close(client);
+    }
+
     public void testApplyRelevantHeaders() throws Exception {
-        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final ThreadContext threadContext = client.threadPool().getThreadContext();
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", true)));
         final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
@@ -139,7 +149,7 @@ public class RestControllerTests extends ESTestCase {
     }
 
     public void testRequestWithDisallowedMultiValuedHeader() {
-        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final ThreadContext threadContext = client.threadPool().getThreadContext();
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", false)));
         final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
@@ -153,10 +163,10 @@ public class RestControllerTests extends ESTestCase {
     }
 
     public void testRequestWithDisallowedMultiValuedHeaderButSameValues() {
-        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final ThreadContext threadContext = client.threadPool().getThreadContext();
         Set<RestHeaderDefinition> headers = new HashSet<>(Arrays.asList(new RestHeaderDefinition("header.1", true),
             new RestHeaderDefinition("header.2", false)));
-        final RestController restController = new RestController(headers, null, null, circuitBreakerService, usageService);
+        final RestController restController = new RestController(headers, null, client, circuitBreakerService, usageService);
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put("header.1", Collections.singletonList("boo"));
         restHeaders.put("header.2", Arrays.asList("foo", "foo"));
@@ -237,11 +247,11 @@ public class RestControllerTests extends ESTestCase {
                 h -> {
                     assertSame(handler, h);
                     return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true);
-                }, null, circuitBreakerService, usageService);
+                }, client, circuitBreakerService, usageService);
         restController.registerHandler(RestRequest.Method.GET, "/wrapped", handler);
         RestRequest request = testRestRequest("/wrapped", "{}", XContentType.JSON);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
-        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(request, channel, client.threadPool().getThreadContext());
         httpServerTransport.start();
         assertTrue(wrapperCalled.get());
         assertFalse(handlerCalled.get());
@@ -253,7 +263,7 @@ public class RestControllerTests extends ESTestCase {
         RestRequest request = testRestRequest("/", content, XContentType.JSON);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.OK);
 
-        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(request, channel, client.threadPool().getThreadContext());
 
         assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
         assertEquals(0, inFlightRequestsBreaker.getUsed());
@@ -265,7 +275,7 @@ public class RestControllerTests extends ESTestCase {
         RestRequest request = testRestRequest("/error", content, XContentType.JSON);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
 
-        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(request, channel, client.threadPool().getThreadContext());
 
         assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
         assertEquals(0, inFlightRequestsBreaker.getUsed());
@@ -278,7 +288,7 @@ public class RestControllerTests extends ESTestCase {
         RestRequest request = testRestRequest("/error", content, XContentType.JSON);
         ExceptionThrowingChannel channel = new ExceptionThrowingChannel(request, true);
 
-        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(request, channel, client.threadPool().getThreadContext());
 
         assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
         assertEquals(0, inFlightRequestsBreaker.getUsed());
@@ -290,7 +300,7 @@ public class RestControllerTests extends ESTestCase {
         RestRequest request = testRestRequest("/", content, XContentType.JSON);
         AssertingChannel channel = new AssertingChannel(request, true, RestStatus.TOO_MANY_REQUESTS);
 
-        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(request, channel, client.threadPool().getThreadContext());
 
         assertEquals(1, inFlightRequestsBreaker.getTrippedCount());
         assertEquals(0, inFlightRequestsBreaker.getUsed());
@@ -306,7 +316,7 @@ public class RestControllerTests extends ESTestCase {
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
 
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(request, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -315,7 +325,7 @@ public class RestControllerTests extends ESTestCase {
         AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
 
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -333,7 +343,7 @@ public class RestControllerTests extends ESTestCase {
         });
 
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -344,7 +354,7 @@ public class RestControllerTests extends ESTestCase {
         AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.NOT_ACCEPTABLE);
 
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -368,7 +378,7 @@ public class RestControllerTests extends ESTestCase {
         });
 
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -393,7 +403,7 @@ public class RestControllerTests extends ESTestCase {
         });
 
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -414,7 +424,7 @@ public class RestControllerTests extends ESTestCase {
         });
 
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -435,7 +445,7 @@ public class RestControllerTests extends ESTestCase {
             }
         });
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -457,7 +467,7 @@ public class RestControllerTests extends ESTestCase {
             }
         });
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
     }
 
@@ -466,7 +476,7 @@ public class RestControllerTests extends ESTestCase {
         final AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.BAD_REQUEST);
         restController.dispatchBadRequest(
             channel,
-            new ThreadContext(Settings.EMPTY),
+            client.threadPool().getThreadContext(),
             randomBoolean() ? new IllegalStateException("bad request") : new Throwable("bad request"));
         assertTrue(channel.getSendResponseCalled());
         assertThat(channel.getRestResponse().content().utf8ToString(), containsString("bad request"));
@@ -475,7 +485,7 @@ public class RestControllerTests extends ESTestCase {
     public void testDispatchBadRequestUnknownCause() {
         final FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build();
         final AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.BAD_REQUEST);
-        restController.dispatchBadRequest(channel, new ThreadContext(Settings.EMPTY), null);
+        restController.dispatchBadRequest(channel, client.threadPool().getThreadContext(), null);
         assertTrue(channel.getSendResponseCalled());
         assertThat(channel.getRestResponse().content().utf8ToString(), containsString("unknown cause"));
     }
@@ -486,7 +496,7 @@ public class RestControllerTests extends ESTestCase {
             .withPath("/favicon.ico")
             .build();
         final AssertingChannel channel = new AssertingChannel(fakeRestRequest, false, RestStatus.OK);
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
         assertThat(channel.getRestResponse().contentType(), containsString("image/x-icon"));
     }
@@ -498,7 +508,7 @@ public class RestControllerTests extends ESTestCase {
             .withPath("/favicon.ico")
             .build();
         final AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.METHOD_NOT_ALLOWED);
-        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
         assertThat(channel.getRestResponse().getHeaders().containsKey("Allow"), equalTo(true));
         assertThat(channel.getRestResponse().getHeaders().get("Allow"), hasItem(equalTo(RestRequest.Method.GET.toString())));
@@ -571,7 +581,7 @@ public class RestControllerTests extends ESTestCase {
 
         final AssertingChannel channel = new AssertingChannel(request, true, RestStatus.METHOD_NOT_ALLOWED);
         assertFalse(channel.getSendResponseCalled());
-        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(request, channel, client.threadPool().getThreadContext());
         assertTrue(channel.getSendResponseCalled());
         assertThat(channel.getRestResponse().getHeaders().containsKey("Allow"), equalTo(true));
         assertThat(channel.getRestResponse().getHeaders().get("Allow"), hasItem(equalTo(RestRequest.Method.GET.toString())));

server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -26,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.analysis.NameOrDefinition;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpNodeClient;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
@@ -95,8 +97,10 @@ public class RestAnalyzeActionTests extends ESTestCase {
         RestAnalyzeAction action = new RestAnalyzeAction();
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
             .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build();
-        IOException e = expectThrows(IOException.class, () -> action.handleRequest(request, null, null));
-        assertThat(e.getMessage(), containsString("expecting double-quote"));
+        try (NodeClient client = new NoOpNodeClient(this.getClass().getSimpleName())) {
+            IOException e = expectThrows(IOException.class, () -> action.handleRequest(request, null, client));
+            assertThat(e.getMessage(), containsString("expecting double-quote"));
+        }
     }
 
     public void testParseXContentForAnalyzeRequestWithUnknownParamThrowsException() throws Exception {

server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingActionTests.java

@@ -50,6 +50,10 @@ public class RestGetFieldMappingActionTests extends RestActionTestCase {
             params.put(INCLUDE_TYPE_NAME_PARAMETER, "false");
             path = "some_index/_mapping/field/some_field";
         }
+
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withMethod(RestRequest.Method.GET)
             .withPath(path)
@@ -76,6 +80,9 @@ public class RestGetFieldMappingActionTests extends RestActionTestCase {
             .withParams(params)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         FakeRestChannel channel = new FakeRestChannel(request, false, 1);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         controller().dispatchRequest(request, channel, threadContext);

server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingActionTests.java

@@ -78,6 +78,9 @@ public class RestGetMappingActionTests extends RestActionTestCase {
             .withParams(params)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         FakeRestChannel channel = new FakeRestChannel(request, false, 1);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         controller().dispatchRequest(request, channel, threadContext);
@@ -98,6 +101,9 @@ public class RestGetMappingActionTests extends RestActionTestCase {
             .withPath("/some_index/_mappings")
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         FakeRestChannel channel = new FakeRestChannel(request, false, 1);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         controller().dispatchRequest(request, channel, threadContext);

server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java

@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.rest.action.admin.indices;
 
-import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
@@ -43,6 +43,7 @@ import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.usage.UsageService;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.util.Collections;
@@ -87,6 +88,12 @@ public class RestValidateQueryActionTests extends AbstractSearchTestCase {
         controller.registerHandler(action);
     }
 
+    @Before
+    public void ensureCleanContext() {
+        // Make sure we have a clean context for each test
+        threadPool.getThreadContext().stashContext();
+    }
+
     @AfterClass
     public static void terminateThreadPool() throws InterruptedException {
         terminate(threadPool);

server/src/test/java/org/elasticsearch/rest/action/document/RestBulkActionTests.java

@@ -19,8 +19,11 @@
 
 package org.elasticsearch.rest.action.document;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -28,15 +31,14 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpNodeClient;
 import org.elasticsearch.test.rest.FakeRestRequest;
-import org.hamcrest.CustomMatcher;
-import org.mockito.Mockito;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.argThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.mockito.Mockito.mock;
 
 /**
@@ -45,32 +47,34 @@ import static org.mockito.Mockito.mock;
 public class RestBulkActionTests extends ESTestCase {
 
     public void testBulkPipelineUpsert() throws Exception {
-        final NodeClient mockClient = mock(NodeClient.class);
-        final Map<String, String> params = new HashMap<>();
-        params.put("pipeline", "timestamps");
-        new RestBulkAction(settings(Version.CURRENT).build())
-            .handleRequest(
-                new FakeRestRequest.Builder(
-                    xContentRegistry()).withPath("my_index/_bulk").withParams(params)
-                    .withContent(
-                        new BytesArray(
-                            "{\"index\":{\"_id\":\"1\"}}\n" +
-                                "{\"field1\":\"val1\"}\n" +
-                                "{\"update\":{\"_id\":\"2\"}}\n" +
-                                "{\"script\":{\"source\":\"ctx._source.counter++;\"},\"upsert\":{\"field1\":\"upserted_val\"}}\n"
-                        ),
-                        XContentType.JSON
-                    ).withMethod(RestRequest.Method.POST).build(),
-                mock(RestChannel.class), mockClient
-            );
-        Mockito.verify(mockClient)
-            .bulk(argThat(new CustomMatcher<BulkRequest>("Pipeline in upsert request") {
-                @Override
-                public boolean matches(final Object item) {
-                    BulkRequest request = (BulkRequest) item;
-                    UpdateRequest update = (UpdateRequest) request.requests().get(1);
-                    return "timestamps".equals(update.upsertRequest().getPipeline());
-                }
-            }), any());
+        SetOnce<Boolean> bulkCalled = new SetOnce<>();
+        try (NodeClient verifyingClient = new NoOpNodeClient(this.getTestName()) {
+            @Override
+            public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
+                bulkCalled.set(true);
+                assertThat(request.requests(), hasSize(2));
+                UpdateRequest updateRequest = (UpdateRequest) request.requests().get(1);
+                assertThat(updateRequest.upsertRequest().getPipeline(), equalTo("timestamps"));
+            }
+        }) {
+            final Map<String, String> params = new HashMap<>();
+            params.put("pipeline", "timestamps");
+            new RestBulkAction(settings(Version.CURRENT).build())
+                .handleRequest(
+                    new FakeRestRequest.Builder(
+                        xContentRegistry()).withPath("my_index/_bulk").withParams(params)
+                        .withContent(
+                            new BytesArray(
+                                "{\"index\":{\"_id\":\"1\"}}\n" +
+                                    "{\"field1\":\"val1\"}\n" +
+                                    "{\"update\":{\"_id\":\"2\"}}\n" +
+                                    "{\"script\":{\"source\":\"ctx._source.counter++;\"},\"upsert\":{\"field1\":\"upserted_val\"}}\n"
+                            ),
+                            XContentType.JSON
+                        ).withMethod(RestRequest.Method.POST).build(),
+                    mock(RestChannel.class), verifyingClient
+                );
+            assertThat(bulkCalled.get(), equalTo(true));
+        }
     }
 }

server/src/test/java/org/elasticsearch/rest/action/document/RestDeleteActionTests.java

@@ -21,8 +21,8 @@ package org.elasticsearch.rest.action.document;
 
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequest.Method;
-import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
 import org.junit.Before;
 
 public class RestDeleteActionTests extends RestActionTestCase {
@@ -33,6 +33,9 @@ public class RestDeleteActionTests extends RestActionTestCase {
     }
 
     public void testTypeInPath() {
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withMethod(Method.DELETE)
             .withPath("/some_index/some_type/some_id")

server/src/test/java/org/elasticsearch/rest/action/document/RestGetActionTests.java

@@ -32,6 +32,9 @@ public class RestGetActionTests extends RestActionTestCase {
     }
 
     public void testTypeInPathWithGet() {
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withPath("/some_index/some_type/some_id");
         dispatchRequest(deprecatedRequest.withMethod(Method.GET).build());
@@ -43,6 +46,9 @@ public class RestGetActionTests extends RestActionTestCase {
     }
 
     public void testTypeInPathWithHead() {
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         FakeRestRequest.Builder deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withPath("/some_index/some_type/some_id");
         dispatchRequest(deprecatedRequest.withMethod(Method.HEAD).build());

server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java

@@ -23,6 +23,7 @@ import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequest.Method;
@@ -65,13 +66,19 @@ public class RestGetSourceActionTests extends RestActionTestCase {
      * test deprecation is logged if type is used in path
      */
     public void testTypeInPath() {
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
         for (Method method : Arrays.asList(Method.GET, Method.HEAD)) {
-            RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+            // Ensure we have a fresh context for each request so we don't get duplicate headers
+            try (ThreadContext.StoredContext ignore = verifyingClient.threadPool().getThreadContext().stashContext()) {
+                RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
                     .withMethod(method)
                     .withPath("/some_index/some_type/id/_source")
                     .build();
-            dispatchRequest(request);
-            assertWarnings(RestGetSourceAction.TYPES_DEPRECATION_MESSAGE);
+
+                dispatchRequest(request);
+                assertWarnings(RestGetSourceAction.TYPES_DEPRECATION_MESSAGE);
+            }
         }
     }
 
@@ -79,9 +86,13 @@ public class RestGetSourceActionTests extends RestActionTestCase {
      * test deprecation is logged if type is used as parameter
      */
     public void testTypeParameter() {
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
         Map<String, String> params = new HashMap<>();
         params.put("type", "some_type");
         for (Method method : Arrays.asList(Method.GET, Method.HEAD)) {
+            // Ensure we have a fresh context for each request so we don't get duplicate headers
+            try (ThreadContext.StoredContext ignore = verifyingClient.threadPool().getThreadContext().stashContext()) {
             RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
                     .withMethod(method)
                     .withPath("/some_index/_source/id")
@@ -89,6 +100,7 @@ public class RestGetSourceActionTests extends RestActionTestCase {
                     .build();
             dispatchRequest(request);
             assertWarnings(RestGetSourceAction.TYPES_DEPRECATION_MESSAGE);
+            }
         }
     }
 

server/src/test/java/org/elasticsearch/rest/action/document/RestIndexActionTests.java

@@ -19,8 +19,8 @@
 
 package org.elasticsearch.rest.action.document;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.ClusterName;
@@ -36,13 +36,11 @@ import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.test.rest.RestActionTestCase;
 import org.junit.Before;
-import org.mockito.ArgumentCaptor;
 
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.verify;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class RestIndexActionTests extends RestActionTestCase {
 
@@ -106,6 +104,13 @@ public class RestIndexActionTests extends RestActionTestCase {
     }
 
     private void checkAutoIdOpType(Version minClusterVersion, DocWriteRequest.OpType expectedOpType) {
+        SetOnce<Boolean> executeCalled = new SetOnce<>();
+        verifyingClient.setExecuteVerifier((actionType, request) -> {
+            assertThat(request, instanceOf(IndexRequest.class));
+            assertThat(((IndexRequest) request).opType(), equalTo(expectedOpType));
+            executeCalled.set(true);
+            return null;
+        });
         RestRequest autoIdRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withMethod(RestRequest.Method.POST)
             .withPath("/some_index/_doc")
@@ -116,9 +121,6 @@ public class RestIndexActionTests extends RestActionTestCase {
                 .add(new DiscoveryNode("test", buildNewFakeTransportAddress(), minClusterVersion))
                 .build()).build());
         dispatchRequest(autoIdRequest);
-        ArgumentCaptor<IndexRequest> argumentCaptor = ArgumentCaptor.forClass(IndexRequest.class);
-        verify(nodeClient).index(argumentCaptor.capture(), any(ActionListener.class));
-        IndexRequest indexRequest = argumentCaptor.getValue();
-        assertEquals(expectedOpType, indexRequest.opType());
+        assertThat(executeCalled.get(), equalTo(true));
     }
 }

server/src/test/java/org/elasticsearch/rest/action/document/RestMultiGetActionTests.java

@@ -38,6 +38,9 @@ public class RestMultiGetActionTests extends RestActionTestCase {
     }
 
     public void testTypeInPath() {
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withMethod(Method.GET)
             .withPath("some_index/some_type/_mget")
@@ -67,6 +70,9 @@ public class RestMultiGetActionTests extends RestActionTestCase {
                 .endArray()
             .endObject();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
             .withPath("_mget")
             .withContent(BytesReference.bytes(content), XContentType.JSON)

server/src/test/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsActionTests.java

@@ -25,8 +25,8 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequest.Method;
-import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -46,6 +46,9 @@ public class RestMultiTermVectorsActionTests extends RestActionTestCase {
             .withPath("/some_index/some_type/_mtermvectors")
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestMultiTermVectorsAction.TYPES_DEPRECATION_MESSAGE);
     }
@@ -60,6 +63,9 @@ public class RestMultiTermVectorsActionTests extends RestActionTestCase {
             .withParams(params)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestMultiTermVectorsAction.TYPES_DEPRECATION_MESSAGE);
     }
@@ -80,6 +86,9 @@ public class RestMultiTermVectorsActionTests extends RestActionTestCase {
             .withContent(BytesReference.bytes(content), XContentType.JSON)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE);
     }

server/src/test/java/org/elasticsearch/rest/action/document/RestTermVectorsActionTests.java

@@ -25,8 +25,8 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequest.Method;
-import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -44,6 +44,9 @@ public class RestTermVectorsActionTests extends RestActionTestCase {
             .withPath("/some_index/some_type/some_id/_termvectors")
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE);
     }
@@ -60,6 +63,9 @@ public class RestTermVectorsActionTests extends RestActionTestCase {
             .withContent(BytesReference.bytes(content), XContentType.JSON)
             .build();
 
+         // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+         verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE);
     }

server/src/test/java/org/elasticsearch/rest/action/document/RestUpdateActionTests.java

@@ -47,6 +47,9 @@ public class RestUpdateActionTests extends RestActionTestCase {
     }
 
     public void testTypeInPath() {
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withMethod(Method.POST)
             .withPath("/some_index/some_type/some_id/_update")

server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java

@@ -21,8 +21,8 @@ package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequest.Method;
-import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
 import org.junit.Before;
 
 import java.util.HashMap;
@@ -41,6 +41,9 @@ public class RestCountActionTests extends RestActionTestCase {
             .withPath("/some_index/some_type/_count")
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestCountAction.TYPES_DEPRECATION_MESSAGE);
     }
@@ -55,6 +58,9 @@ public class RestCountActionTests extends RestActionTestCase {
             .withParams(params)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestCountAction.TYPES_DEPRECATION_MESSAGE);
     }

server/src/test/java/org/elasticsearch/rest/action/search/RestExplainActionTests.java

@@ -32,6 +32,9 @@ public class RestExplainActionTests extends RestActionTestCase {
     }
 
     public void testTypeInPath() {
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteVerifier((arg1, arg2) -> null);
+
         RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withMethod(RestRequest.Method.GET)
             .withPath("/some_index/some_type/some_id/_explain")

server/src/test/java/org/elasticsearch/rest/action/search/RestMultiSearchActionTests.java

@@ -23,8 +23,8 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
 import org.junit.Before;
 
 import java.nio.charset.StandardCharsets;
@@ -46,6 +46,9 @@ public class RestMultiSearchActionTests extends RestActionTestCase {
             .withContent(bytesContent, XContentType.JSON)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteLocallyVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestMultiSearchAction.TYPES_DEPRECATION_MESSAGE);
     }
@@ -60,6 +63,9 @@ public class RestMultiSearchActionTests extends RestActionTestCase {
             .withContent(bytesContent, XContentType.JSON)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteLocallyVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestMultiSearchAction.TYPES_DEPRECATION_MESSAGE);
     }

server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java

@@ -20,8 +20,8 @@
 package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
 import org.junit.Before;
 
 import java.util.HashMap;
@@ -40,6 +40,9 @@ public class RestSearchActionTests extends RestActionTestCase {
             .withPath("/some_index/some_type/_search")
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteLocallyVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE);
     }
@@ -54,6 +57,9 @@ public class RestSearchActionTests extends RestActionTestCase {
             .withParams(params)
             .build();
 
+        // We're not actually testing anything to do with the client, but need to set this so it doesn't fail the test for being unset.
+        verifyingClient.setExecuteLocallyVerifier((arg1, arg2) -> null);
+
         dispatchRequest(request);
         assertWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE);
     }

server/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java

@@ -19,26 +19,24 @@
 
 package org.elasticsearch.search.scroll;
 
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestClearScrollAction;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpNodeClient;
 import org.elasticsearch.test.rest.FakeRestChannel;
 import org.elasticsearch.test.rest.FakeRestRequest;
-import org.mockito.ArgumentCaptor;
 
 import java.util.Collections;
-import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
+import static org.hamcrest.Matchers.hasSize;
 
 public class RestClearScrollActionTests extends ESTestCase {
 
@@ -51,21 +49,23 @@ public class RestClearScrollActionTests extends ESTestCase {
     }
 
     public void testBodyParamsOverrideQueryStringParams() throws Exception {
-        NodeClient nodeClient = mock(NodeClient.class);
-        doNothing().when(nodeClient).searchScroll(any(), any());
-
-        RestClearScrollAction action = new RestClearScrollAction();
-        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+        SetOnce<Boolean> scrollCalled = new SetOnce<>();
+        try (NodeClient nodeClient = new NoOpNodeClient(this.getTestName()) {
+            @Override
+            public void clearScroll(ClearScrollRequest request, ActionListener<ClearScrollResponse> listener) {
+                scrollCalled.set(true);
+                assertThat(request.getScrollIds(), hasSize(1));
+                assertThat(request.getScrollIds().get(0), equalTo("BODY"));
+            }
+        }) {
+            RestClearScrollAction action = new RestClearScrollAction();
+            RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
                 .withParams(Collections.singletonMap("scroll_id", "QUERY_STRING"))
                 .withContent(new BytesArray("{\"scroll_id\": [\"BODY\"]}"), XContentType.JSON).build();
-        FakeRestChannel channel = new FakeRestChannel(request, false, 0);
-        action.handleRequest(request, channel, nodeClient);
+            FakeRestChannel channel = new FakeRestChannel(request, false, 0);
+            action.handleRequest(request, channel, nodeClient);
 
-        ArgumentCaptor<ClearScrollRequest> argument = ArgumentCaptor.forClass(ClearScrollRequest.class);
-        verify(nodeClient).clearScroll(argument.capture(), anyObject());
-        ClearScrollRequest clearScrollRequest = argument.getValue();
-        List<String> scrollIds = clearScrollRequest.getScrollIds();
-        assertEquals(1, scrollIds.size());
-        assertEquals("BODY", scrollIds.get(0));
+            assertThat(scrollCalled.get(), equalTo(true));
+        }
     }
 }

server/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java

@@ -19,6 +19,9 @@
 
 package org.elasticsearch.search.scroll;
 
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -26,19 +29,14 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestSearchScrollAction;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpNodeClient;
 import org.elasticsearch.test.rest.FakeRestChannel;
 import org.elasticsearch.test.rest.FakeRestRequest;
-import org.mockito.ArgumentCaptor;
 
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
 
 public class RestSearchScrollActionTests extends ESTestCase {
 
@@ -51,23 +49,26 @@ public class RestSearchScrollActionTests extends ESTestCase {
     }
 
     public void testBodyParamsOverrideQueryStringParams() throws Exception {
-        NodeClient nodeClient = mock(NodeClient.class);
-        doNothing().when(nodeClient).searchScroll(any(), any());
-
-        RestSearchScrollAction action = new RestSearchScrollAction();
-        Map<String, String> params = new HashMap<>();
-        params.put("scroll_id", "QUERY_STRING");
-        params.put("scroll", "1000m");
-        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+        SetOnce<Boolean> scrollCalled = new SetOnce<>();
+        try (NodeClient nodeClient = new NoOpNodeClient(this.getTestName()) {
+            @Override
+            public void searchScroll(SearchScrollRequest request, ActionListener<SearchResponse> listener) {
+                scrollCalled.set(true);
+                assertThat(request.scrollId(), equalTo("BODY"));
+                assertThat(request.scroll().keepAlive().getStringRep(), equalTo("1m"));
+            }
+        }) {
+            RestSearchScrollAction action = new RestSearchScrollAction();
+            Map<String, String> params = new HashMap<>();
+            params.put("scroll_id", "QUERY_STRING");
+            params.put("scroll", "1000m");
+            RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
                 .withParams(params)
                 .withContent(new BytesArray("{\"scroll_id\":\"BODY\", \"scroll\":\"1m\"}"), XContentType.JSON).build();
-        FakeRestChannel channel = new FakeRestChannel(request, false, 0);
-        action.handleRequest(request, channel, nodeClient);
+            FakeRestChannel channel = new FakeRestChannel(request, false, 0);
+            action.handleRequest(request, channel, nodeClient);
 
-        ArgumentCaptor<SearchScrollRequest> argument = ArgumentCaptor.forClass(SearchScrollRequest.class);
-        verify(nodeClient).searchScroll(argument.capture(), anyObject());
-        SearchScrollRequest searchScrollRequest = argument.getValue();
-        assertEquals("BODY", searchScrollRequest.scrollId());
-        assertEquals("1m", searchScrollRequest.scroll().keepAlive().getStringRep());
+            assertThat(scrollCalled.get(), equalTo(true));
+        }
     }
 }

server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java

@@ -65,6 +65,8 @@ import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAct
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction;
 import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction;
 import org.elasticsearch.action.bulk.BulkAction;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.IndexingPressure;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.TransportBulkAction;
@@ -1481,7 +1483,8 @@ public class SnapshotResiliencyTests extends ESTestCase {
                     },
                     a -> node, null, emptySet()
                 );
-                final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
+                final IndexNameExpressionResolver indexNameExpressionResolver =
+                    new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
                 repositoriesService = new RepositoriesService(
                     settings, clusterService, transportService,
                     Collections.singletonMap(FsRepository.TYPE, getRepoFactory(environment)), emptyMap(), threadPool

server/src/test/java/org/elasticsearch/usage/UsageServiceTests.java

@@ -24,6 +24,7 @@ import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.search.aggregations.support.AggregationUsageService;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpNodeClient;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.util.Collections;
@@ -104,20 +105,22 @@ public class UsageServiceTests extends ESTestCase {
         usageService.addRestHandler(handlerD);
         usageService.addRestHandler(handlerE);
         usageService.addRestHandler(handlerF);
-        handlerA.handleRequest(restRequest, null, null);
-        handlerB.handleRequest(restRequest, null, null);
-        handlerA.handleRequest(restRequest, null, null);
-        handlerA.handleRequest(restRequest, null, null);
-        handlerB.handleRequest(restRequest, null, null);
-        handlerC.handleRequest(restRequest, null, null);
-        handlerC.handleRequest(restRequest, null, null);
-        handlerD.handleRequest(restRequest, null, null);
-        handlerA.handleRequest(restRequest, null, null);
-        handlerB.handleRequest(restRequest, null, null);
-        handlerE.handleRequest(restRequest, null, null);
-        handlerF.handleRequest(restRequest, null, null);
-        handlerC.handleRequest(restRequest, null, null);
-        handlerD.handleRequest(restRequest, null, null);
+        try (NodeClient client = new NoOpNodeClient(this.getClass().getSimpleName() + "TestClient")) {
+            handlerA.handleRequest(restRequest, null, client);
+            handlerB.handleRequest(restRequest, null, client);
+            handlerA.handleRequest(restRequest, null, client);
+            handlerA.handleRequest(restRequest, null, client);
+            handlerB.handleRequest(restRequest, null, client);
+            handlerC.handleRequest(restRequest, null, client);
+            handlerC.handleRequest(restRequest, null, client);
+            handlerD.handleRequest(restRequest, null, client);
+            handlerA.handleRequest(restRequest, null, client);
+            handlerB.handleRequest(restRequest, null, client);
+            handlerE.handleRequest(restRequest, null, client);
+            handlerF.handleRequest(restRequest, null, client);
+            handlerC.handleRequest(restRequest, null, client);
+            handlerD.handleRequest(restRequest, null, client);
+        }
         Map<String, Long> restUsage = usageService.getRestUsageStats();
         assertThat(restUsage, notNullValue());
         assertThat(restUsage.size(), equalTo(6));

test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java

@@ -20,10 +20,10 @@
 package org.elasticsearch.test.client;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.client.support.AbstractClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -32,7 +32,10 @@ import org.elasticsearch.threadpool.ThreadPool;
 import java.util.concurrent.TimeUnit;
 
 /**
- * Client that always responds with {@code null} to every request. Override this for testing.
+ * Client that always responds with {@code null} to every request. Override {@link #doExecute(ActionType, ActionRequest, ActionListener)}
+ * for testing.
+ *
+ * See also {@link NoOpNodeClient} if you need to mock a {@link org.elasticsearch.client.node.NodeClient}.
  */
 public class NoOpClient extends AbstractClient {
     /**
@@ -43,7 +46,7 @@ public class NoOpClient extends AbstractClient {
     }
 
     /**
-     * Create a new {@link TestThreadPool} for this client.
+     * Create a new {@link TestThreadPool} for this client. This {@linkplain TestThreadPool} is terminated on {@link #close()}.
      */
     public NoOpClient(String testName) {
         super(Settings.EMPTY, new TestThreadPool(testName));

test/framework/src/main/java/org/elasticsearch/test/client/NoOpNodeClient.java

@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.client;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskListener;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterService;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+/**
+ * Client that always response with {@code null} to every request. Override {@link #doExecute(ActionType, ActionRequest, ActionListener)},
+ * {@link #executeLocally(ActionType, ActionRequest, ActionListener)}, or {@link #executeLocally(ActionType, ActionRequest, TaskListener)}
+ * for testing.
+ *
+ * See also {@link NoOpClient} if you do not specifically need a {@link NodeClient}.
+ */
+public class NoOpNodeClient extends NodeClient {
+
+    /**
+     * Build with {@link ThreadPool}. This {@linkplain ThreadPool} is terminated on {@link #close()}.
+     */
+    public NoOpNodeClient(ThreadPool threadPool) {
+        super(Settings.EMPTY, threadPool);
+    }
+
+    /**
+     * Create a new {@link TestThreadPool} for this client. This {@linkplain TestThreadPool} is terminated on {@link #close()}.
+     */
+    public NoOpNodeClient(String testName) {
+        super(Settings.EMPTY, new TestThreadPool(testName));
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse>
+    void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
+        listener.onResponse(null);
+    }
+
+    @Override
+    public void initialize(Map<ActionType, TransportAction> actions, Supplier<String> localNodeId,
+                           RemoteClusterService remoteClusterService, NamedWriteableRegistry namedWriteableRegistry) {
+        throw new UnsupportedOperationException("cannot initialize " + this.getClass().getSimpleName());
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse>
+    Task executeLocally(ActionType<Response> action, Request request, ActionListener<Response> listener) {
+        listener.onResponse(null);
+        return null;
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse>
+    Task executeLocally(ActionType<Response> action, Request request, TaskListener<Response> listener) {
+        listener.onResponse(null, null);
+        return null;
+    }
+
+    @Override
+    public String getLocalNodeId() {
+        return null;
+    }
+
+    @Override
+    public Client getRemoteClusterClient(String clusterAlias) {
+        return null;
+    }
+
+    @Override
+    public void close() {
+        try {
+            ThreadPool.terminate(threadPool(), 10, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            throw new ElasticsearchException(e.getMessage(), e);
+        }
+    }
+}

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -677,6 +677,20 @@ public abstract class ESRestTestCase extends ESTestCase {
         try {
             final Request deleteRequest = new Request("DELETE", "*");
             deleteRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : ""));
+            RequestOptions.Builder allowSystemIndexAccessWarningOptions = RequestOptions.DEFAULT.toBuilder();
+            allowSystemIndexAccessWarningOptions.setWarningsHandler(warnings -> {
+                    if (warnings.size() == 0) {
+                        return false;
+                    } else if (warnings.size() > 1) {
+                        return true;
+                    }
+                    // We don't know exactly which indices we're cleaning up in advance, so just accept all system index access warnings.
+                    final String warning = warnings.get(0);
+                    final boolean isSystemIndexWarning = warning.contains("this request accesses system indices")
+                        && warning.contains("but in a future major version, direct access to system indices will be prevented by default");
+                    return isSystemIndexWarning == false;
+                });
+            deleteRequest.setOptions(allowSystemIndexAccessWarningOptions);
             final Response response = adminClient().performRequest(deleteRequest);
             try (InputStream is = response.getEntity().getContent()) {
                 assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged"));
@@ -818,7 +832,19 @@ public abstract class ESRestTestCase extends ESTestCase {
     protected void refreshAllIndices() throws IOException {
         boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0);
         Request refreshRequest = new Request("POST", "/_refresh");
-        refreshRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : ""));
+        refreshRequest.addParameter("expand_wildcards", "open" + (includeHidden ? ",hidden" : ""));
+        // Allow system index deprecation warnings
+        final Builder requestOptions = RequestOptions.DEFAULT.toBuilder();
+        requestOptions.setWarningsHandler(warnings -> {
+            if (warnings.isEmpty()) {
+                return false;
+            } else if (warnings.size() > 1) {
+                return true;
+            } else {
+                return warnings.get(0).startsWith("this request accesses system indices:") == false;
+            }
+        });
+        refreshRequest.setOptions(requestOptions);
         client().performRequest(refreshRequest);
     }
 

test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java

@@ -19,19 +19,26 @@
 
 package org.elasticsearch.test.rest;
 
-import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskListener;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpNodeClient;
 import org.elasticsearch.usage.UsageService;
+import org.junit.After;
 import org.junit.Before;
 
 import java.util.Collections;
-
-import static org.mockito.Mockito.mock;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
 
 /**
  * A common base class for Rest*ActionTests. Provides access to a {@link RestController}
@@ -39,17 +46,22 @@ import static org.mockito.Mockito.mock;
  */
 public abstract class RestActionTestCase extends ESTestCase {
     private RestController controller;
-    protected NodeClient nodeClient;
+    protected VerifyingClient verifyingClient;
 
     @Before
     public void setUpController() {
-        nodeClient = mock(NodeClient.class);
+        verifyingClient = new VerifyingClient(this.getTestName());
         controller = new RestController(Collections.emptySet(), null,
-            nodeClient,
+            verifyingClient,
             new NoneCircuitBreakerService(),
             new UsageService());
     }
 
+    @After
+    public void tearDownController() {
+        verifyingClient.close();
+    }
+
     /**
      * A test {@link RestController}. This controller can be used to register and delegate
      * to handlers, but uses a mock client and cannot carry out the full request.
@@ -66,4 +78,76 @@ public abstract class RestActionTestCase extends ESTestCase {
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         controller.dispatchRequest(request, channel, threadContext);
     }
+
+    /**
+     * A mocked {@link org.elasticsearch.client.node.NodeClient} which can be easily reconfigured to verify arbitrary verification
+     * functions, and can be reset to allow reconfiguration partway through a test without having to construct a new object.
+     *
+     * By default, will throw {@link AssertionError} when any execution method is called, unless configured otherwise using
+     * {@link #setExecuteVerifier(BiFunction)} or {@link #setExecuteLocallyVerifier(BiFunction)}.
+     */
+    public static class VerifyingClient extends NoOpNodeClient {
+        AtomicReference<BiFunction> executeVerifier = new AtomicReference<>();
+        AtomicReference<BiFunction> executeLocallyVerifier = new AtomicReference<>();
+
+        public VerifyingClient(String testName) {
+            super(testName);
+            reset();
+        }
+
+        /**
+         * Clears any previously set verifier functions set by {@link #setExecuteVerifier(BiFunction)} and/or
+         * {@link #setExecuteLocallyVerifier(BiFunction)}. These functions are replaced with functions which will throw an
+         * {@link AssertionError} if called.
+         */
+        public void reset() {
+            executeVerifier.set((arg1, arg2) -> {
+                throw new AssertionError();
+            });
+            executeLocallyVerifier.set((arg1, arg2) -> {
+                throw new AssertionError();
+            });
+        }
+
+        /**
+         * Sets the function that will be called when {@link #doExecute(ActionType, ActionRequest, ActionListener)} is called. The given
+         * function should return either a subclass of {@link ActionResponse} or {@code null}.
+         * @param verifier A function which is called in place of {@link #doExecute(ActionType, ActionRequest, ActionListener)}
+         */
+        public <Request extends ActionRequest, Response extends ActionResponse>
+        void setExecuteVerifier(BiFunction<ActionType<Response>, Request, Void> verifier) {
+            executeVerifier.set(verifier);
+        }
+
+        @Override
+        public <Request extends ActionRequest, Response extends ActionResponse>
+        void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
+            listener.onResponse((Response) executeVerifier.get().apply(action, request));
+        }
+
+        /**
+         * Sets the function that will be called when {@link #executeLocally(ActionType, ActionRequest, TaskListener)}is called. The given
+         * function should return either a subclass of {@link ActionResponse} or {@code null}.
+         * @param verifier A function which is called in place of {@link #executeLocally(ActionType, ActionRequest, TaskListener)}
+         */
+        public <Request extends ActionRequest, Response extends ActionResponse>
+        void setExecuteLocallyVerifier(BiFunction<ActionType<Response>, Request, Void> verifier) {
+            executeLocallyVerifier.set(verifier);
+        }
+
+        @Override
+        public <Request extends ActionRequest, Response extends ActionResponse>
+        Task executeLocally(ActionType<Response> action, Request request, ActionListener<Response> listener) {
+            listener.onResponse((Response) executeLocallyVerifier.get().apply(action, request));
+            return null;
+        }
+
+        @Override
+        public <Request extends ActionRequest, Response extends ActionResponse>
+        Task executeLocally(ActionType<Response> action, Request request, TaskListener<Response> listener) {
+            listener.onResponse(null, (Response) executeLocallyVerifier.get().apply(action, request));
+            return null;
+        }
+
+    }
 }

x-pack/docs/en/watcher/managing-watches.asciidoc

@@ -33,4 +33,4 @@ GET .watches/_search
   "size" : 100
 }
 --------------------------------------------------
-// TEST[setup:my_active_watch]
+// TEST[skip:deprecation warning]

x-pack/docs/en/watcher/troubleshooting.asciidoc

@@ -18,7 +18,7 @@ do that by submitting the following request:
 --------------------------------------------------
 GET .watches/_mapping
 --------------------------------------------------
-// TEST[setup:my_active_watch]
+// TEST[skip:deprecation warning]
 
 If the index mappings are missing, follow these steps to restore the correct
 mappings:
@@ -33,7 +33,7 @@ mappings:
 --------------------------------------------------
 DELETE .watches
 --------------------------------------------------
-// TEST[skip:index deletion]
+// TEST[skip:index deletion and deprecation warning]
 --
 . Disable direct access to the `.watches` index:
 .. Stop the Elasticsearch node.

x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchActionTests.java

@@ -5,81 +5,65 @@
  */
 package org.elasticsearch.xpack.search;
 
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.client.node.NodeClient;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestChannel;
 import org.elasticsearch.test.rest.FakeRestRequest;
-import org.elasticsearch.usage.UsageService;
+import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest;
 import org.junit.Before;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
 
-public class RestSubmitAsyncSearchActionTests extends ESTestCase {
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class RestSubmitAsyncSearchActionTests extends RestActionTestCase {
 
     private RestSubmitAsyncSearchAction action;
-    private ActionRequest lastCapturedRequest;
-    private RestController controller;
-    private NodeClient nodeClient;
 
     @Before
-    public void setUpController() {
-        nodeClient = new NodeClient(Settings.EMPTY, null) {
-
-            @Override
-            public <Request extends ActionRequest, Response extends ActionResponse> Task executeLocally(ActionType<Response> action,
-                    Request request, ActionListener<Response> listener) {
-                lastCapturedRequest = request;
-                return new Task(1L, "type", "action", "description", null, null);
-            }
-        };
-        nodeClient.initialize(new HashMap<>(), () -> "local", null, new NamedWriteableRegistry(Collections.emptyList()));
-        controller = new RestController(Collections.emptySet(), null,
-            nodeClient,
-            new NoneCircuitBreakerService(),
-            new UsageService());
+    public void setUpAction() {
         action = new RestSubmitAsyncSearchAction();
-        controller.registerHandler(action);
+        controller().registerHandler(action);
     }
 
     /**
      * Check that the appropriate defaults are set on the {@link SubmitAsyncSearchRequest} if
      * no parameters are specified on the rest request itself.
      */
+    @SuppressWarnings("unchecked")
     public void testRequestParameterDefaults() throws IOException {
-            RestRequest submitAsyncRestRequest = new FakeRestRequest.Builder(xContentRegistry())
-                .withMethod(RestRequest.Method.POST)
-                .withPath("/test_index/_async_search")
-                .withContent(new BytesArray("{}"), XContentType.JSON)
-                .build();
-            dispatchRequest(submitAsyncRestRequest);
-            SubmitAsyncSearchRequest submitRequest = (SubmitAsyncSearchRequest) lastCapturedRequest;
-            assertEquals(TimeValue.timeValueSeconds(1), submitRequest.getWaitForCompletionTimeout());
-            assertFalse(submitRequest.isKeepOnCompletion());
-            assertEquals(TimeValue.timeValueDays(5), submitRequest.getKeepAlive());
+        SetOnce<Boolean> executeCalled = new SetOnce<>();
+        verifyingClient.setExecuteLocallyVerifier((actionType, request) -> {
+            assertThat(request, instanceOf(SubmitAsyncSearchRequest.class));
+            SubmitAsyncSearchRequest submitRequest = (SubmitAsyncSearchRequest) request;
+            assertThat(submitRequest.getWaitForCompletionTimeout(), equalTo(TimeValue.timeValueSeconds(1)));
+            assertThat(submitRequest.isKeepOnCompletion(), equalTo(false));
+            assertThat(submitRequest.getKeepAlive(), equalTo(TimeValue.timeValueDays(5)));
             // check parameters we implicitly set in the SubmitAsyncSearchRequest ctor
-            assertFalse(submitRequest.getSearchRequest().isCcsMinimizeRoundtrips());
-            assertEquals(5, submitRequest.getSearchRequest().getBatchedReduceSize());
-            assertEquals(true, submitRequest.getSearchRequest().requestCache());
-            assertEquals(1, submitRequest.getSearchRequest().getPreFilterShardSize().intValue());
+            assertThat(submitRequest.getSearchRequest().isCcsMinimizeRoundtrips(), equalTo(false));
+            assertThat(submitRequest.getSearchRequest().getBatchedReduceSize(), equalTo(5));
+            assertThat(submitRequest.getSearchRequest().requestCache(), equalTo(true));
+            assertThat(submitRequest.getSearchRequest().getPreFilterShardSize().intValue(), equalTo(1));
+            executeCalled.set(true);
+            return null;
+        });
+        RestRequest submitAsyncRestRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withMethod(RestRequest.Method.POST)
+            .withPath("/test_index/_async_search")
+            .withContent(new BytesArray("{}"), XContentType.JSON)
+            .build();
+        dispatchRequest(submitAsyncRestRequest);
+        assertThat(executeCalled.get(), equalTo(true));
     }
 
     public void testParameters() throws IOException {
@@ -100,15 +84,26 @@ public class RestSubmitAsyncSearchActionTests extends ESTestCase {
 
     private <T> void doTestParameter(String paramName, String paramValue, T expectedValue,
             Function<SubmitAsyncSearchRequest, T> valueAccessor) {
+        SetOnce<Boolean> executeCalled = new SetOnce<>();
+        verifyingClient.setExecuteLocallyVerifier((actionType, request) -> {
+            assertThat(request, instanceOf(SubmitAsyncSearchRequest.class));
+            assertThat(valueAccessor.apply((SubmitAsyncSearchRequest) request), equalTo(expectedValue));
+            executeCalled.set(true);
+            return null;
+        });
         Map<String, String> params = new HashMap<>();
         params.put(paramName, paramValue);
         RestRequest submitAsyncRestRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST)
-                .withPath("/test_index/_async_search")
-                .withParams(params)
-                .withContent(new BytesArray("{}"), XContentType.JSON).build();
-        dispatchRequest(submitAsyncRestRequest);
-        SubmitAsyncSearchRequest submitRequest = (SubmitAsyncSearchRequest) lastCapturedRequest;
-        assertEquals(expectedValue, valueAccessor.apply(submitRequest));
+            .withPath("/test_index/_async_search")
+            .withParams(params)
+            .withContent(new BytesArray("{}"), XContentType.JSON).build();
+
+        // Get a new context each time, so we don't get exceptions due to trying to add the same header multiple times
+        try (ThreadContext.StoredContext context = verifyingClient.threadPool().getThreadContext().stashContext()) {
+            dispatchRequest(submitAsyncRestRequest);
+        }
+        assertThat(executeCalled.get(), equalTo(true));
+        verifyingClient.reset();
     }
 
     /**
@@ -117,6 +112,6 @@ public class RestSubmitAsyncSearchActionTests extends ESTestCase {
     protected void dispatchRequest(RestRequest request) {
         FakeRestChannel channel = new FakeRestChannel(request, false, 1);
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        controller.dispatchRequest(request, channel, threadContext);
+        controller().dispatchRequest(request, channel, threadContext);
     }
 }