Run Node deprecation checks locally (#38065) (#38250)

At times, we need to check for usage of deprecated settings in settings
which should not be returned by the NodeInfo API.  This commit changes
the deprecation info API to run all node checks locally so that these
settings can be checked without exposing them via any externally
accessible API.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java

@@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
@@ -35,7 +34,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -62,6 +60,23 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
         return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList());
     }
 
+    private static List<DeprecationIssue> mergeNodeIssues(NodesDeprecationCheckResponse response) {
+        Map<DeprecationIssue, List<String>> issueListMap = new HashMap<>();
+        for (NodesDeprecationCheckAction.NodeResponse resp : response.getNodes()) {
+            for (DeprecationIssue issue : resp.getDeprecationIssues()) {
+                issueListMap.computeIfAbsent(issue, (key) -> new ArrayList<>()).add(resp.getNode().getName());
+            }
+        }
+
+        return issueListMap.entrySet().stream()
+            .map(entry -> {
+                DeprecationIssue issue = entry.getKey();
+                String details = issue.getDetails() != null ? issue.getDetails() + " " : "";
+                return new DeprecationIssue(issue.getLevel(), issue.getMessage(), issue.getUrl(),
+                    details + "(nodes impacted: " + entry.getValue() + ")");
+            }).collect(Collectors.toList());
+    }
+
     @Override
     public Response newResponse() {
         return new Response();
@@ -159,32 +174,29 @@ public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response
          * this function will run through all the checks and build out the final list of issues that exist in the
          * cluster.
          *
-         * @param nodesInfo The list of {@link NodeInfo} metadata objects for retrieving node-level information
-         * @param nodesStats The list of {@link NodeStats} metadata objects for retrieving node-level information
          * @param state The cluster state
          * @param indexNameExpressionResolver Used to resolve indices into their concrete names
          * @param indices The list of index expressions to evaluate using `indexNameExpressionResolver`
          * @param indicesOptions The options to use when resolving and filtering which indices to check
          * @param datafeeds The ml datafeed configurations
-         * @param clusterSettingsChecks The list of cluster-level checks
-         * @param nodeSettingsChecks The list of node-level checks
+         * @param nodeDeprecationResponse The response containing the deprecation issues found on each node
          * @param indexSettingsChecks The list of index-level checks that will be run across all specified
          *                            concrete indices
+         * @param clusterSettingsChecks The list of cluster-level checks
          * @param mlSettingsCheck The list of ml checks
          * @return The list of deprecation issues found in the cluster
          */
-        public static DeprecationInfoAction.Response from(List<NodeInfo> nodesInfo, List<NodeStats> nodesStats, ClusterState state,
-                                                  IndexNameExpressionResolver indexNameExpressionResolver,
-                                                  String[] indices, IndicesOptions indicesOptions,
-                                                  List<DatafeedConfig> datafeeds,
-                                                  List<Function<ClusterState,DeprecationIssue>>clusterSettingsChecks,
-                                                  List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks,
-                                                  List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
-                                                  List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
+        public static DeprecationInfoAction.Response from(ClusterState state,
+                                                          IndexNameExpressionResolver indexNameExpressionResolver,
+                                                          String[] indices, IndicesOptions indicesOptions,
+                                                          List<DatafeedConfig> datafeeds,
+                                                          NodesDeprecationCheckResponse nodeDeprecationResponse,
+                                                          List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks,
+                                                          List<Function<ClusterState, DeprecationIssue>> clusterSettingsChecks,
+                                                          List<Function<DatafeedConfig, DeprecationIssue>> mlSettingsCheck) {
             List<DeprecationIssue> clusterSettingsIssues = filterChecks(clusterSettingsChecks,
                 (c) -> c.apply(state));
-            List<DeprecationIssue> nodeSettingsIssues = filterChecks(nodeSettingsChecks,
-                (c) -> c.apply(nodesInfo, nodesStats));
+            List<DeprecationIssue> nodeSettingsIssues = mergeNodeIssues(nodeDeprecationResponse);
             List<DeprecationIssue> mlSettingsIssues = new ArrayList<>();
             for (DatafeedConfig config : datafeeds) {
                 mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config)));

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.support.nodes.BaseNodeRequest;
+import org.elasticsearch.action.support.nodes.BaseNodeResponse;
+import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Runs deprecation checks on each node. Deprecation checks are performed locally so that filtered settings
+ * can be accessed in the deprecation checks.
+ */
+public class NodesDeprecationCheckAction extends Action<NodesDeprecationCheckResponse> {
+    public static final NodesDeprecationCheckAction INSTANCE = new NodesDeprecationCheckAction();
+    public static final String NAME = "cluster:admin/xpack/deprecation/nodes/info";
+
+    private NodesDeprecationCheckAction() {
+        super(NAME);
+    }
+
+    @Override
+    public NodesDeprecationCheckResponse newResponse() {
+        return new NodesDeprecationCheckResponse();
+    }
+
+    public static class NodeRequest extends BaseNodeRequest {
+
+        NodesDeprecationCheckRequest request;
+
+        public NodeRequest() {}
+        public NodeRequest(String nodeId, NodesDeprecationCheckRequest request) {
+            super(nodeId);
+            this.request = request;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            request = new NodesDeprecationCheckRequest();
+            request.readFrom(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            request.writeTo(out);
+        }
+    }
+
+    public static class NodeResponse extends BaseNodeResponse {
+        private List<DeprecationIssue> deprecationIssues;
+
+        public NodeResponse() {
+            super();
+        }
+
+        public NodeResponse(DiscoveryNode node, List<DeprecationIssue> deprecationIssues) {
+            super(node);
+            this.deprecationIssues = deprecationIssues;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            deprecationIssues = in.readList(DeprecationIssue::new);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeList(this.deprecationIssues);
+        }
+
+        public static NodeResponse readNodeResponse(StreamInput in) throws IOException {
+            NodeResponse nodeResponse = new NodeResponse();
+            nodeResponse.readFrom(in);
+            return nodeResponse;
+        }
+
+        public List<DeprecationIssue> getDeprecationIssues() {
+            return deprecationIssues;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NodeResponse that = (NodeResponse) o;
+            return Objects.equals(getDeprecationIssues(), that.getDeprecationIssues())
+                && Objects.equals(getNode(), that.getNode());
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getNode(), getDeprecationIssues());
+        }
+    }
+
+    public static class RequestBuilder extends NodesOperationRequestBuilder<NodesDeprecationCheckRequest,
+        NodesDeprecationCheckResponse, RequestBuilder> {
+
+        protected RequestBuilder(ElasticsearchClient client,
+                                 Action<NodesDeprecationCheckResponse> action,
+                                 NodesDeprecationCheckRequest request) {
+            super(client, action, request);
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.action.support.nodes.BaseNodesRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+public class NodesDeprecationCheckRequest extends BaseNodesRequest<NodesDeprecationCheckRequest> {
+    public NodesDeprecationCheckRequest() {}
+
+    public NodesDeprecationCheckRequest(String... nodesIds) {
+        super(nodesIds);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash((Object[]) this.nodesIds());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        NodesDeprecationCheckRequest that = (NodesDeprecationCheckRequest) obj;
+        return Arrays.equals(this.nodesIds(), that.nodesIds());
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.support.nodes.BaseNodesResponse;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+public class NodesDeprecationCheckResponse extends BaseNodesResponse<NodesDeprecationCheckAction.NodeResponse> {
+
+    public NodesDeprecationCheckResponse() {}
+
+    public NodesDeprecationCheckResponse(ClusterName clusterName,
+                                         List<NodesDeprecationCheckAction.NodeResponse> nodes,
+                                         List<FailedNodeException> failures) {
+        super(clusterName, nodes, failures);
+    }
+
+    @Override
+    protected List<NodesDeprecationCheckAction.NodeResponse> readNodesFrom(StreamInput in) throws IOException {
+        return in.readList(NodesDeprecationCheckAction.NodeResponse::readNodeResponse);
+    }
+
+    @Override
+    protected void writeNodesTo(StreamOutput out, List<NodesDeprecationCheckAction.NodeResponse> nodes) throws IOException {
+        out.writeStreamableList(nodes);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NodesDeprecationCheckResponse that = (NodesDeprecationCheckResponse) o;
+        return Objects.equals(getClusterName(), that.getClusterName())
+            && Objects.equals(getNodes(), that.getNodes())
+            && Objects.equals(failures(), that.failures());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getClusterName(), getNodes(), failures());
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java

@@ -5,10 +5,7 @@
  */
 package org.elasticsearch.xpack.core.deprecation;
 
-import org.elasticsearch.Build;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -31,7 +28,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -78,12 +74,6 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
         DiscoveryNode discoveryNode = DiscoveryNode.createLocal(Settings.EMPTY,
             new TransportAddress(TransportAddress.META_ADDRESS, 9300), "test");
         ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(metadata).build();
-        List<NodeInfo> nodeInfos = Collections.singletonList(new NodeInfo(Version.CURRENT, Build.CURRENT,
-            discoveryNode, null, null, null, null,
-            null, null, null, null, null, null));
-        List<NodeStats> nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null,
-            null, null, null, null, null, null, null, null,
-            null, null, null, null));
         List<DatafeedConfig> datafeeds = Collections.singletonList(DatafeedConfigTests.createRandomizedDatafeedConfig("foo"));
         IndexNameExpressionResolver resolver = new IndexNameExpressionResolver();
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false,
@@ -97,11 +87,6 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
             Collections.unmodifiableList(Arrays.asList(
                 (s) -> clusterIssueFound ? foundIssue : null
             ));
-        List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> nodeSettingsChecks =
-            Collections.unmodifiableList(Arrays.asList(
-                (ln, ls) -> nodeIssueFound ? foundIssue : null
-            ));
-
         List<Function<IndexMetaData, DeprecationIssue>> indexSettingsChecks =
             Collections.unmodifiableList(Arrays.asList(
                 (idx) -> indexIssueFound ? foundIssue : null
@@ -111,9 +96,17 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
                         (idx) -> mlIssueFound ? foundIssue : null
                 ));
 
-        DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(nodeInfos, nodeStats, state,
+        NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse(
+            new ClusterName(randomAlphaOfLength(5)),
+            nodeIssueFound
+                ? Collections.singletonList(
+                    new NodesDeprecationCheckAction.NodeResponse(discoveryNode, Collections.singletonList(foundIssue)))
+                : Collections.emptyList(),
+            Collections.emptyList());
+
+        DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(state,
             resolver, Strings.EMPTY_ARRAY, indicesOptions, datafeeds,
-            clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks, mlSettingsChecks);
+            nodeDeprecationIssues, indexSettingsChecks, clusterSettingsChecks, mlSettingsChecks);
 
         if (clusterIssueFound) {
             assertThat(response.getClusterSettingsIssues(), equalTo(Collections.singletonList(foundIssue)));
@@ -122,7 +115,10 @@ public class DeprecationInfoActionResponseTests extends AbstractStreamableTestCa
         }
 
         if (nodeIssueFound) {
-            assertThat(response.getNodeSettingsIssues(), equalTo(Collections.singletonList(foundIssue)));
+            String details = foundIssue.getDetails() != null ? foundIssue.getDetails() + " " : "";
+            DeprecationIssue mergedFoundIssue = new DeprecationIssue(foundIssue.getLevel(), foundIssue.getMessage(), foundIssue.getUrl(),
+                details + "(nodes impacted: [" + discoveryNode.getName() + "])");
+            assertThat(response.getNodeSettingsIssues(), equalTo(Collections.singletonList(mergedFoundIssue)));
         } else {
             assertTrue(response.getNodeSettingsIssues().isEmpty());
         }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.test.AbstractStreamableTestCase;
+
+import java.io.IOException;
+
+public class NodesDeprecationCheckRequestTests
+    extends AbstractStreamableTestCase<NodesDeprecationCheckRequest> {
+
+    @Override
+    protected NodesDeprecationCheckRequest createBlankInstance() {
+        return new NodesDeprecationCheckRequest();
+    }
+
+    @Override
+    protected NodesDeprecationCheckRequest mutateInstance(NodesDeprecationCheckRequest instance) throws IOException {
+        int newSize = randomValueOtherThan(instance.nodesIds().length, () -> randomIntBetween(0,10));
+        String[] newNodeIds = randomArray(newSize, newSize, String[]::new, () -> randomAlphaOfLengthBetween(5, 10));
+        return new NodesDeprecationCheckRequest(newNodeIds);
+    }
+
+    @Override
+    protected NodesDeprecationCheckRequest createTestInstance() {
+        return new NodesDeprecationCheckRequest(randomArray(0, 10, String[]::new,
+            ()-> randomAlphaOfLengthBetween(5,10)));
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.deprecation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class NodesDeprecationCheckResponseTests
+    extends AbstractStreamableTestCase<NodesDeprecationCheckResponse> {
+
+    @Override
+    protected NodesDeprecationCheckResponse createBlankInstance() {
+        return new NodesDeprecationCheckResponse();
+    }
+
+    @Override
+    protected NodesDeprecationCheckResponse createTestInstance() {
+
+        List<NodesDeprecationCheckAction.NodeResponse> responses =
+            Arrays.asList(randomArray(1, 10, NodesDeprecationCheckAction.NodeResponse[]::new,
+                NodesDeprecationCheckResponseTests::randomNodeResponse));
+        return new NodesDeprecationCheckResponse(new ClusterName(randomAlphaOfLength(10)),
+            responses,
+            Collections.emptyList());
+    }
+
+    @Override
+    protected NodesDeprecationCheckResponse mutateInstance(NodesDeprecationCheckResponse instance) throws IOException {
+        int mutate = randomIntBetween(1,3);
+        switch (mutate) {
+            case 1:
+                List<NodesDeprecationCheckAction.NodeResponse> responses = new ArrayList<>(instance.getNodes());
+                responses.add(randomNodeResponse());
+                return new NodesDeprecationCheckResponse(instance.getClusterName(), responses, instance.failures());
+            case 2:
+                ArrayList<FailedNodeException> failures = new ArrayList<>(instance.failures());
+                failures.add(new FailedNodeException("test node", "test failure", new RuntimeException(randomAlphaOfLength(10))));
+                return new NodesDeprecationCheckResponse(instance.getClusterName(), instance.getNodes(), failures);
+            case 3:
+                String clusterName = randomValueOtherThan(instance.getClusterName().value(), () -> randomAlphaOfLengthBetween(5,15));
+                return new NodesDeprecationCheckResponse(new ClusterName(clusterName), instance.getNodes(), instance.failures());
+            default:
+                fail("invalid mutation");
+        }
+
+        return super.mutateInstance(instance);
+    }
+
+    private static DiscoveryNode randomDiscoveryNode() throws Exception {
+        InetAddress inetAddress = InetAddress.getByAddress(randomAlphaOfLength(5),
+            new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1});
+        TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535));
+
+        return new DiscoveryNode(randomAlphaOfLength(5), randomAlphaOfLength(5), transportAddress,
+            Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
+    }
+
+    private static NodesDeprecationCheckAction.NodeResponse randomNodeResponse() {
+        DiscoveryNode node;
+        try {
+            node = randomDiscoveryNode();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        List<DeprecationIssue> issuesList = Arrays.asList(randomArray(0,10, DeprecationIssue[]::new,
+            DeprecationIssueTests::createTestInstance));
+        return new NodesDeprecationCheckAction.NodeResponse(node, issuesList);
+    }
+}

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java

@@ -19,7 +19,9 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
+import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Supplier;
@@ -30,7 +32,10 @@ import java.util.function.Supplier;
 public class Deprecation extends Plugin implements ActionPlugin {
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        return Collections.singletonList(new ActionHandler<>(DeprecationInfoAction.INSTANCE, TransportDeprecationInfoAction.class));
+        return Collections.unmodifiableList(Arrays.asList(
+            new ActionHandler<>(DeprecationInfoAction.INSTANCE, TransportDeprecationInfoAction.class),
+            new ActionHandler<>(NodesDeprecationCheckAction.INSTANCE, TransportNodeDeprecationCheckAction.class)
+        ));
     }
 
     @Override

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.deprecation;
 
-import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -33,7 +33,7 @@ public class DeprecationChecks {
     static List<Function<ClusterState, DeprecationIssue>> CLUSTER_SETTINGS_CHECKS =
         Collections.emptyList();
 
-    static List<BiFunction<List<NodeInfo>, List<NodeStats>, DeprecationIssue>> NODE_SETTINGS_CHECKS =
+    static List<BiFunction<Settings, PluginsAndModules, DeprecationIssue>> NODE_SETTINGS_CHECKS =
         Collections.unmodifiableList(Arrays.asList(
             // STUB
         ));

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java

@@ -5,11 +5,10 @@
  */
 package org.elasticsearch.xpack.deprecation;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.client.node.NodeClient;
@@ -20,7 +19,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -29,14 +27,22 @@ import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
+import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction;
+import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckRequest;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS;
+import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
+import static org.elasticsearch.xpack.deprecation.DeprecationChecks.ML_SETTINGS_CHECKS;
 
 public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction<DeprecationInfoAction.Request,
         DeprecationInfoAction.Response> {
+    private static final Logger logger = LogManager.getLogger(TransportDeprecationInfoAction.class);
 
     private final XPackLicenseState licenseState;
     private final NodeClient client;
@@ -76,39 +82,32 @@ public class TransportDeprecationInfoAction extends TransportMasterNodeReadActio
     protected final void masterOperation(final DeprecationInfoAction.Request request, ClusterState state,
                                          final ActionListener<DeprecationInfoAction.Response> listener) {
         if (licenseState.isDeprecationAllowed()) {
-            NodesInfoRequest nodesInfoRequest = new NodesInfoRequest("_local").settings(true).plugins(true);
-            NodesStatsRequest nodesStatsRequest = new NodesStatsRequest("_local").fs(true);
 
-            final ThreadContext threadContext = client.threadPool().getThreadContext();
-            ClientHelper.executeAsyncWithOrigin(threadContext, ClientHelper.DEPRECATION_ORIGIN, nodesInfoRequest,
-                    ActionListener.<NodesInfoResponse>wrap(
-                    nodesInfoResponse -> {
-                        if (nodesInfoResponse.hasFailures()) {
-                            throw nodesInfoResponse.failures().get(0);
-                        }
-                        ClientHelper.executeAsyncWithOrigin(threadContext, ClientHelper.DEPRECATION_ORIGIN, nodesStatsRequest,
-                                ActionListener.<NodesStatsResponse>wrap(
-                                    nodesStatsResponse -> {
-                                        if (nodesStatsResponse.hasFailures()) {
-                                            throw nodesStatsResponse.failures().get(0);
-                                        }
+            NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all");
+            ClientHelper.executeAsyncWithOrigin(client, ClientHelper.DEPRECATION_ORIGIN,
+                NodesDeprecationCheckAction.INSTANCE, nodeDepReq,
+                ActionListener.wrap(response -> {
+                if (response.hasFailures()) {
+                    List<String> failedNodeIds = response.failures().stream()
+                        .map(failure -> failure.nodeId() + ": " + failure.getMessage())
+                        .collect(Collectors.toList());
+                    logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds);
+                    for (FailedNodeException failure : response.failures()) {
+                        logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure);
+                    }
+                }
+                getDatafeedConfigs(ActionListener.wrap(
+                    datafeeds -> {
+                        listener.onResponse(
+                            DeprecationInfoAction.Response.from(state, indexNameExpressionResolver,
+                                request.indices(), request.indicesOptions(), datafeeds,
+                                response, INDEX_SETTINGS_CHECKS, CLUSTER_SETTINGS_CHECKS,
+                                ML_SETTINGS_CHECKS));
+                    },
+                    listener::onFailure
+                ));
 
-                                        getDatafeedConfigs(ActionListener.wrap(
-                                            datafeeds -> {
-                                                listener.onResponse(
-                                                        DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(),
-                                                        nodesStatsResponse.getNodes(), state, indexNameExpressionResolver,
-                                                        request.indices(), request.indicesOptions(), datafeeds,
-                                                        DeprecationChecks.CLUSTER_SETTINGS_CHECKS,
-                                                        DeprecationChecks.NODE_SETTINGS_CHECKS,
-                                                        DeprecationChecks.INDEX_SETTINGS_CHECKS,
-                                                        DeprecationChecks.ML_SETTINGS_CHECKS));
-                                             },
-                                             listener::onFailure
-                                        ));
-                                    }, listener::onFailure),
-                                client.admin().cluster()::nodesStats);
-                    }, listener::onFailure), client.admin().cluster()::nodesInfo);
+            }, listener::onFailure));
         } else {
             listener.onFailure(LicenseUtils.newComplianceException(XPackField.DEPRECATION));
         }

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.deprecation;
+
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.nodes.TransportNodesAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
+import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction;
+import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckRequest;
+import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckResponse;
+
+import java.util.List;
+
+public class TransportNodeDeprecationCheckAction extends TransportNodesAction<NodesDeprecationCheckRequest,
+    NodesDeprecationCheckResponse,
+    NodesDeprecationCheckAction.NodeRequest,
+    NodesDeprecationCheckAction.NodeResponse> {
+
+    private final Settings settings;
+    private final PluginsService pluginsService;
+
+    @Inject
+    public TransportNodeDeprecationCheckAction(Settings settings, ThreadPool threadPool,
+                                               ClusterService clusterService, TransportService transportService,
+                                               PluginsService pluginsService, ActionFilters actionFilters) {
+        super(NodesDeprecationCheckAction.NAME, threadPool, clusterService, transportService, actionFilters,
+            NodesDeprecationCheckRequest::new,
+            NodesDeprecationCheckAction.NodeRequest::new,
+            ThreadPool.Names.GENERIC,
+            NodesDeprecationCheckAction.NodeResponse.class);
+        this.settings = settings;
+        this.pluginsService = pluginsService;
+    }
+
+    @Override
+    protected NodesDeprecationCheckResponse newResponse(NodesDeprecationCheckRequest request,
+                                                        List<NodesDeprecationCheckAction.NodeResponse> nodeResponses,
+                                                        List<FailedNodeException> failures) {
+        return new NodesDeprecationCheckResponse(clusterService.getClusterName(), nodeResponses, failures);
+    }
+
+    @Override
+    protected NodesDeprecationCheckAction.NodeRequest newNodeRequest(String nodeId, NodesDeprecationCheckRequest request) {
+        return new NodesDeprecationCheckAction.NodeRequest(nodeId, request);
+    }
+
+    @Override
+    protected NodesDeprecationCheckAction.NodeResponse newNodeResponse() {
+        return new NodesDeprecationCheckAction.NodeResponse();
+    }
+
+    @Override
+    protected NodesDeprecationCheckAction.NodeResponse nodeOperation(NodesDeprecationCheckAction.NodeRequest request) {
+        List<DeprecationIssue> issues = DeprecationInfoAction.filterChecks(DeprecationChecks.NODE_SETTINGS_CHECKS,
+            (c) -> c.apply(settings, pluginsService.info()));
+
+        return new NodesDeprecationCheckAction.NodeResponse(transportService.getLocalNode(), issues);
+    }
+
+
+}