Add `coordinating_only` node selector (#30313)

Today we can execute cluster API actions on only master, data or ingest nodes
using the `master:true`, `data:true` and `ingest:true` filters, but it is not
so easy to select coordinating-only nodes (i.e. those nodes that are neither
master nor data nor ingest nodes). This change fixes this by adding support for
a `coordinating_only` filter such that `coordinating_only:true` adds all
coordinating-only nodes to the set of selected nodes, and 
`coordinating_only:false` deletes them.

Resolves #28831.

docs/CHANGELOG.asciidoc

@@ -176,6 +176,9 @@ started or stopped. ({pull}30118[#30118])
 
 Added put index template API to the high level rest client ({pull}30400[#30400])
 
+Add ability to filter coordinating-only nodes when interacting with cluster
+APIs. ({pull}30313[#30313])
+
 [float]
 === Bug Fixes
 

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java

@@ -44,6 +44,8 @@ import java.util.function.Predicate;
  */
 public class DiscoveryNode implements Writeable, ToXContentFragment {
 
+    static final String COORDINATING_ONLY = "coordinating_only";
+
     public static boolean nodeRequiresLocalStorage(Settings settings) {
         boolean localStorageEnable = Node.NODE_LOCAL_STORAGE_SETTING.get(settings);
         if (localStorageEnable == false &&

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java

@@ -148,6 +148,19 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
         return nodes.build();
     }
 
+    /**
+     * Get a {@link Map} of the coordinating only nodes (nodes which are neither master, nor data, nor ingest nodes) arranged by their ids
+     *
+     * @return {@link Map} of the coordinating only nodes arranged by their ids
+     */
+    public ImmutableOpenMap<String, DiscoveryNode> getCoordinatingOnlyNodes() {
+        ImmutableOpenMap.Builder<String, DiscoveryNode> nodes = ImmutableOpenMap.builder(this.nodes);
+        nodes.removeAll(masterNodes.keys());
+        nodes.removeAll(dataNodes.keys());
+        nodes.removeAll(ingestNodes.keys());
+        return nodes.build();
+    }
+
     /**
      * Get a node by its id
      *
@@ -297,7 +310,7 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
      * - "_local" or "_master" for the relevant nodes
      * - a node id
      * - a wild card pattern that will be matched against node names
-     * - a "attr:value" pattern, where attr can be a node role (master, data, ingest etc.) in which case the value can be true of false
+     * - a "attr:value" pattern, where attr can be a node role (master, data, ingest etc.) in which case the value can be true or false,
      *   or a generic node attribute name in which case value will be treated as a wildcard and matched against the node attribute values.
      */
     public String[] resolveNodes(String... nodes) {
@@ -349,6 +362,12 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
                             } else {
                                 resolvedNodesIds.removeAll(ingestNodes.keys());
                             }
+                        } else if (DiscoveryNode.COORDINATING_ONLY.equals(matchAttrName)) {
+                            if (Booleans.parseBoolean(matchAttrValue, true)) {
+                                resolvedNodesIds.addAll(getCoordinatingOnlyNodes().keys());
+                            } else {
+                                resolvedNodesIds.removeAll(getCoordinatingOnlyNodes().keys());
+                            }
                         } else {
                             for (DiscoveryNode node : this) {
                                 for (Map.Entry<String, String> entry : node.getAttributes().entrySet()) {

server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java

@@ -102,9 +102,17 @@ public class DiscoveryNodesTests extends ESTestCase {
                     .map(DiscoveryNode::getId)
                     .toArray(String[]::new);
 
-        assertThat(
+        final String[] nonCoordinatorOnlyNodes =
-                discoveryNodes.resolveNodes("_all", "data:false", "ingest:false", "master:false"),
+                StreamSupport.stream(discoveryNodes.getNodes().values().spliterator(), false)
+                    .map(n -> n.value)
+                    .filter(n -> n.isMasterNode() || n.isDataNode() || n.isIngestNode())
+                    .map(DiscoveryNode::getId)
+                    .toArray(String[]::new);
+
+        assertThat(discoveryNodes.resolveNodes("coordinating_only:true"), arrayContainingInAnyOrder(coordinatorOnlyNodes));
+        assertThat(discoveryNodes.resolveNodes("_all", "data:false", "ingest:false", "master:false"),
             arrayContainingInAnyOrder(coordinatorOnlyNodes));
+        assertThat(discoveryNodes.resolveNodes("_all", "coordinating_only:false"), arrayContainingInAnyOrder(nonCoordinatorOnlyNodes));
     }
 
     public void testResolveNodesIds() {
@@ -275,6 +283,13 @@ public class DiscoveryNodesTests extends ESTestCase {
                 nodes.getIngestNodes().keysIt().forEachRemaining(ids::add);
                 return ids;
             }
+        }, COORDINATING_ONLY(DiscoveryNode.COORDINATING_ONLY + ":true") {
+            @Override
+            Set<String> matchingNodeIds(DiscoveryNodes nodes) {
+                Set<String> ids = new HashSet<>();
+                nodes.getCoordinatingOnlyNodes().keysIt().forEachRemaining(ids::add);
+                return ids;
+            }
         }, CUSTOM_ATTRIBUTE("attr:value") {
             @Override
             Set<String> matchingNodeIds(DiscoveryNodes nodes) {