Fix the support of RestClient Node Sniffer for version 2.x and update tests (#3487)

Fix the support of RestClient Node Sniffer for OpenSearch 2.x, and update unit tests for OpenSearch.
The current code contains the logic to be compatible with Elasticsearch 2.x version, which is conflict with OpenSearch 2.x, so removed that part of legacy code.

* Update the script create_test_nodes_info.bash to dump the response of Nodes Info API GET _nodes/http for OpenSearch 1.0 and 2.0 version, which used for unit test.
* Remove the support of Elasticsearch version 2.x for the Sniffer
* Update unit test to validate the Sniffer compatible with OpenSearch 1.x and 2.x
* Update the API response parser to meet the array notation (in ES 6.1 and above) for the node attributes setting. It will result the value of `node.attr` setting will not be parsed as array in the Sniffer, when using the Sniffer on cluster in Elasticsearch 6.0 and above.
* Replace "master" node role with "cluster_manager" in unit test

Signed-off-by: Tianli Feng <ftianli@amazon.com>

client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java

@@ -49,6 +49,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -241,74 +242,23 @@ public final class OpenSearchNodesSniffer implements NodesSniffer {
         }
 
         Map<String, List<String>> realAttributes = new HashMap<>(protoAttributes.size());
-        List<String> keys = new ArrayList<>(protoAttributes.keySet());
-        for (String key : keys) {
-            if (key.endsWith(".0")) {
-                String realKey = key.substring(0, key.length() - 2);
-                List<String> values = new ArrayList<>();
-                int i = 0;
-                while (true) {
-                    String value = protoAttributes.remove(realKey + "." + i);
-                    if (value == null) {
-                        break;
-                    }
-                    values.add(value);
-                    i++;
-                }
-                realAttributes.put(realKey, unmodifiableList(values));
-            }
-        }
         for (Map.Entry<String, String> entry : protoAttributes.entrySet()) {
+            if (entry.getValue().startsWith("[")) {
+                // Convert string array to list
+                String value = entry.getValue();
+                String[] values = value.substring(1, value.length() - 1).split(", ");
+                realAttributes.put(entry.getKey(), unmodifiableList(Arrays.asList(values)));
+            } else {
                 realAttributes.put(entry.getKey(), singletonList(entry.getValue()));
             }
+        }
 
-        if (version.startsWith("2.")) {
-            /*
-             * 2.x doesn't send roles, instead we try to read them from
-             * attributes.
-             */
-            boolean clientAttribute = v2RoleAttributeValue(realAttributes, "client", false);
-            Boolean masterAttribute = v2RoleAttributeValue(realAttributes, "master", null);
-            Boolean dataAttribute = v2RoleAttributeValue(realAttributes, "data", null);
-            if ((masterAttribute == null && false == clientAttribute) || masterAttribute) {
-                roles.add("master");
-            }
-            if ((dataAttribute == null && false == clientAttribute) || dataAttribute) {
-                roles.add("data");
-            }
-        } else {
         assert sawRoles : "didn't see roles for [" + nodeId + "]";
-        }
         assert boundHosts.contains(publishedHost) : "[" + nodeId + "] doesn't make sense! publishedHost should be in boundHosts";
         logger.trace("adding node [" + nodeId + "]");
         return new Node(publishedHost, boundHosts, name, version, new Roles(roles), unmodifiableMap(realAttributes));
     }
 
-    /**
-     * Returns {@code defaultValue} if the attribute didn't come back,
-     * {@code true} or {@code false} if it did come back as
-     * either of those, or throws an IOException if the attribute
-     * came back in a strange way.
-     */
-    private static Boolean v2RoleAttributeValue(Map<String, List<String>> attributes, String name, Boolean defaultValue)
-        throws IOException {
-        List<String> valueList = attributes.remove(name);
-        if (valueList == null) {
-            return defaultValue;
-        }
-        if (valueList.size() != 1) {
-            throw new IOException("expected only a single attribute value for [" + name + "] but got " + valueList);
-        }
-        switch (valueList.get(0)) {
-            case "true":
-                return true;
-            case "false":
-                return false;
-            default:
-                throw new IOException("expected [" + name + "] to be either [true] or [false] but was [" + valueList.get(0) + "]");
-        }
-    }
-
     /**
      * The supported host schemes.
      */

client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java

@@ -45,8 +45,8 @@ import org.opensearch.client.sniff.OpenSearchNodesSniffer.Scheme;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -85,59 +85,31 @@ public class OpenSearchNodesSnifferParseTests extends RestClientTestCase {
         }
     }
 
+    public void test1x() throws IOException {
+        checkFile(
+            "1.0.0_nodes_http.json",
+            node(9200, "m1", "1.0.0", "master", "ingest"),
+            node(9201, "m2", "1.0.0", "master", "data", "ingest"),
+            node(9202, "m3", "1.0.0", "master", "ingest"),
+            node(9203, "d1", "1.0.0", "data", "ingest"),
+            node(9204, "d2", "1.0.0", "data", "ingest"),
+            node(9205, "d3", "1.0.0", "data", "ingest"),
+            node(9206, "c1", "1.0.0", "ingest"),
+            node(9207, "c2", "1.0.0", "ingest")
+        );
+    }
+
     public void test2x() throws IOException {
         checkFile(
             "2.0.0_nodes_http.json",
-            node(9200, "m1", "2.0.0", true, false, false),
+            node(9200, "m1", "2.0.0", "cluster_manager", "ingest"),
-            node(9201, "m2", "2.0.0", true, true, false),
+            node(9201, "m2", "2.0.0", "cluster_manager", "data", "ingest"),
-            node(9202, "m3", "2.0.0", true, false, false),
+            node(9202, "m3", "2.0.0", "cluster_manager", "ingest"),
-            node(9203, "d1", "2.0.0", false, true, false),
+            node(9203, "d1", "2.0.0", "data", "ingest"),
-            node(9204, "d2", "2.0.0", false, true, false),
+            node(9204, "d2", "2.0.0", "data", "ingest"),
-            node(9205, "d3", "2.0.0", false, true, false),
+            node(9205, "d3", "2.0.0", "data", "ingest"),
-            node(9206, "c1", "2.0.0", false, false, false),
+            node(9206, "c1", "2.0.0", "ingest"),
-            node(9207, "c2", "2.0.0", false, false, false)
+            node(9207, "c2", "2.0.0", "ingest")
-        );
-    }
-
-    public void test5x() throws IOException {
-        checkFile(
-            "5.0.0_nodes_http.json",
-            node(9200, "m1", "5.0.0", true, false, true),
-            node(9201, "m2", "5.0.0", true, true, true),
-            node(9202, "m3", "5.0.0", true, false, true),
-            node(9203, "d1", "5.0.0", false, true, true),
-            node(9204, "d2", "5.0.0", false, true, true),
-            node(9205, "d3", "5.0.0", false, true, true),
-            node(9206, "c1", "5.0.0", false, false, true),
-            node(9207, "c2", "5.0.0", false, false, true)
-        );
-    }
-
-    public void test6x() throws IOException {
-        checkFile(
-            "6.0.0_nodes_http.json",
-            node(9200, "m1", "6.0.0", true, false, true),
-            node(9201, "m2", "6.0.0", true, true, true),
-            node(9202, "m3", "6.0.0", true, false, true),
-            node(9203, "d1", "6.0.0", false, true, true),
-            node(9204, "d2", "6.0.0", false, true, true),
-            node(9205, "d3", "6.0.0", false, true, true),
-            node(9206, "c1", "6.0.0", false, false, true),
-            node(9207, "c2", "6.0.0", false, false, true)
-        );
-    }
-
-    public void test7x() throws IOException {
-        checkFile(
-            "7.3.0_nodes_http.json",
-            node(9200, "m1", "7.3.0", "master", "ingest"),
-            node(9201, "m2", "7.3.0", "master", "data", "ingest"),
-            node(9202, "m3", "7.3.0", "master", "ingest"),
-            node(9203, "d1", "7.3.0", "data", "ingest", "ml"),
-            node(9204, "d2", "7.3.0", "data", "ingest"),
-            node(9205, "d3", "7.3.0", "data", "ingest"),
-            node(9206, "c1", "7.3.0", "ingest"),
-            node(9207, "c2", "7.3.0", "ingest")
         );
     }
 
@@ -163,20 +135,6 @@ public class OpenSearchNodesSnifferParseTests extends RestClientTestCase {
         assertEquals("http", nodes.get(0).getHost().getSchemeName());
     }
 
-    private Node node(int port, String name, String version, boolean master, boolean data, boolean ingest) {
-        final Set<String> roles = new TreeSet<>();
-        if (master) {
-            roles.add("master");
-        }
-        if (data) {
-            roles.add("data");
-        }
-        if (ingest) {
-            roles.add("ingest");
-        }
-        return node(port, name, version, roles);
-    }
-
     private Node node(int port, String name, String version, String... roles) {
         return node(port, name, version, new TreeSet<>(Arrays.asList(roles)));
     }
@@ -184,11 +142,15 @@ public class OpenSearchNodesSnifferParseTests extends RestClientTestCase {
     private Node node(int port, String name, String version, Set<String> roles) {
         HttpHost host = new HttpHost("127.0.0.1", port);
         Set<HttpHost> boundHosts = new HashSet<>(2);
-        boundHosts.add(host);
         boundHosts.add(new HttpHost("[::1]", port));
-        Map<String, List<String>> attributes = new HashMap<>();
+        boundHosts.add(host);
+        Map<String, List<String>> attributes = new LinkedHashMap<>(); // LinkedHashMap to preserve insertion order
         attributes.put("dummy", singletonList("everyone_has_me"));
         attributes.put("number", singletonList(name.substring(1)));
+        if (!version.startsWith("1.0") && !version.startsWith("1.1")) {
+            // Shard Indexing Pressure feature is added in version 1.2.0
+            attributes.put("shard_indexing_pressure_enabled", singletonList(Boolean.TRUE.toString()));
+        }
         attributes.put("array", Arrays.asList(name.substring(0, 1), name.substring(1)));
         return new Node(host, boundHosts, name, version, new Roles(new TreeSet<>(roles)), attributes);
     }

client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java

@@ -234,7 +234,7 @@ public class OpenSearchNodesSnifferTests extends RestClientTestCase {
 
             final Set<String> nodeRoles = new TreeSet<>();
             if (randomBoolean()) {
-                nodeRoles.add("master");
+                nodeRoles.add("cluster_manager");
             }
             if (randomBoolean()) {
                 nodeRoles.add("data");
@@ -283,12 +283,12 @@ public class OpenSearchNodesSnifferTests extends RestClientTestCase {
                 generator.writeEndObject();
             }
 
-            List<String> roles = Arrays.asList(new String[] { "master", "data", "ingest" });
+            List<String> roles = Arrays.asList(new String[] { "cluster_manager", "data", "ingest" });
             Collections.shuffle(roles, getRandom());
             generator.writeArrayFieldStart("roles");
             for (String role : roles) {
-                if ("master".equals(role) && node.getRoles().isMasterEligible()) {
+                if ("cluster_manager".equals(role) && node.getRoles().isMasterEligible()) {
-                    generator.writeString("master");
+                    generator.writeString("cluster_manager");
                 }
                 if ("data".equals(role) && node.getRoles().isData()) {
                     generator.writeString("data");
@@ -307,13 +307,7 @@ public class OpenSearchNodesSnifferTests extends RestClientTestCase {
             if (numAttributes > 0) {
                 generator.writeObjectFieldStart("attributes");
                 for (Map.Entry<String, List<String>> entry : attributes.entrySet()) {
-                    if (entry.getValue().size() == 1) {
+                    generator.writeStringField(entry.getKey(), entry.getValue().toString());
-                        generator.writeStringField(entry.getKey(), entry.getValue().get(0));
-                    } else {
-                        for (int v = 0; v < entry.getValue().size(); v++) {
-                            generator.writeStringField(entry.getKey() + "." + v, entry.getValue().get(v));
-                        }
-                    }
                 }
                 generator.writeEndObject();
             }

client/sniffer/src/test/resources/1.0.0_nodes_http.json

@@ -11,17 +11,17 @@
       "transport_address": "127.0.0.1:9300",
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
-      "version": "7.3.0",
+      "version": "1.0.0",
-      "build_hash": "8f0685b",
+      "build_type": "tar",
+      "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3",
       "roles": [
-        "master",
+        "ingest",
-        "ingest"
+        "master"
       ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "1",
-        "array.0": "m",
+        "array": "[m, 1]"
-        "array.1": "1"
       },
       "http": {
         "bound_address": [
@@ -37,18 +37,18 @@
       "transport_address": "127.0.0.1:9301",
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
-      "version": "7.3.0",
+      "version": "1.0.0",
-      "build_hash": "8f0685b",
+      "build_type": "tar",
+      "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3",
       "roles": [
-        "master",
         "data",
-        "ingest"
+        "ingest",
+        "master"
       ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "2",
-        "array.0": "m",
+        "array": "[m, 2]"
-        "array.1": "2"
       },
       "http": {
         "bound_address": [
@@ -64,17 +64,17 @@
       "transport_address": "127.0.0.1:9302",
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
-      "version": "7.3.0",
+      "version": "1.0.0",
-      "build_hash": "8f0685b",
+      "build_type": "tar",
+      "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3",
       "roles": [
-        "master",
+        "ingest",
-        "ingest"
+        "master"
       ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "3",
-        "array.0": "m",
+        "array": "[m, 3]"
-        "array.1": "3"
       },
       "http": {
         "bound_address": [
@@ -90,18 +90,17 @@
       "transport_address": "127.0.0.1:9303",
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
-      "version": "7.3.0",
+      "version": "1.0.0",
-      "build_hash": "8f0685b",
+      "build_type": "tar",
+      "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3",
       "roles": [
         "data",
-        "ingest",
+        "ingest"
-        "ml"
       ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "1",
-        "array.0": "d",
+        "array": "[d, 1]"
-        "array.1": "1"
       },
       "http": {
         "bound_address": [
@@ -117,8 +116,9 @@
       "transport_address": "127.0.0.1:9304",
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
-      "version": "7.3.0",
+      "version": "1.0.0",
-      "build_hash": "8f0685b",
+      "build_type": "tar",
+      "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3",
       "roles": [
         "data",
         "ingest"
@@ -126,8 +126,7 @@
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "2",
-        "array.0": "d",
+        "array": "[d, 2]"
-        "array.1": "2"
       },
       "http": {
         "bound_address": [
@@ -143,8 +142,9 @@
       "transport_address": "127.0.0.1:9305",
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
-      "version": "7.3.0",
+      "version": "1.0.0",
-      "build_hash": "8f0685b",
+      "build_type": "tar",
+      "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3",
       "roles": [
         "data",
         "ingest"
@@ -152,8 +152,7 @@
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "3",
-        "array.0": "d",
+        "array": "[d, 3]"
-        "array.1": "3"
       },
       "http": {
         "bound_address": [
@@ -169,16 +168,16 @@
       "transport_address": "127.0.0.1:9306",
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
-      "version": "7.3.0",
+      "version": "1.0.0",
-      "build_hash": "8f0685b",
+      "build_type": "tar",
+      "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3",
       "roles": [
         "ingest"
       ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "1",
-        "array.0": "c",
+        "array": "[c, 1]"
-        "array.1": "1"
       },
       "http": {
         "bound_address": [
@@ -194,16 +193,16 @@
       "transport_address": "127.0.0.1:9307",
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
-      "version": "7.3.0",
+      "version": "1.0.0",
-      "build_hash": "8f0685b",
+      "build_type": "tar",
+      "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3",
       "roles": [
         "ingest"
       ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "2",
-        "array.0": "c",
+        "array": "[c, 2]"
-        "array.1": "2"
       },
       "http": {
         "bound_address": [

client/sniffer/src/test/resources/2.0.0_nodes_http.json

@@ -1,4 +1,9 @@
 {
+  "_nodes": {
+    "total": 8,
+    "successful": 8,
+    "failed": 0
+  },
   "cluster_name": "opensearch",
   "nodes": {
     "qr-SOrELSaGW8SlU8nflBw": {
@@ -7,20 +12,22 @@
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
       "version": "2.0.0",
-      "build": "de54438",
+      "build_type": "tar",
-      "http_address": "127.0.0.1:9200",
+      "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0",
+      "roles": [
+        "cluster_manager",
+        "ingest"
+      ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "1",
-        "array.0": "m",
+        "array": "[m, 1]",
-        "data": "false",
+        "shard_indexing_pressure_enabled": "true"
-        "array.1": "1",
-        "master": "true"
       },
       "http": {
         "bound_address": [
-          "127.0.0.1:9200",
+          "[::1]:9200",
-          "[::1]:9200"
+          "127.0.0.1:9200"
         ],
         "publish_address": "127.0.0.1:9200",
         "max_content_length_in_bytes": 104857600
@@ -32,19 +39,23 @@
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
       "version": "2.0.0",
-      "build": "de54438",
+      "build_type": "tar",
-      "http_address": "127.0.0.1:9201",
+      "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0",
+      "roles": [
+        "cluster_manager",
+        "data",
+        "ingest"
+      ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "2",
-        "array.0": "m",
+        "shard_indexing_pressure_enabled": "true",
-        "array.1": "2",
+        "array": "[m, 2]"
-        "master": "true"
       },
       "http": {
         "bound_address": [
-          "127.0.0.1:9201",
+          "[::1]:9201",
-          "[::1]:9201"
+          "127.0.0.1:9201"
         ],
         "publish_address": "127.0.0.1:9201",
         "max_content_length_in_bytes": 104857600
@@ -56,20 +67,22 @@
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
       "version": "2.0.0",
-      "build": "de54438",
+      "build_type": "tar",
-      "http_address": "127.0.0.1:9202",
+      "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0",
+      "roles": [
+        "cluster_manager",
+        "ingest"
+      ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "3",
-        "array.0": "m",
+        "shard_indexing_pressure_enabled": "true",
-        "data": "false",
+        "array": "[m, 3]"
-        "array.1": "3",
-        "master": "true"
       },
       "http": {
         "bound_address": [
-          "127.0.0.1:9202",
+          "[::1]:9202",
-          "[::1]:9202"
+          "127.0.0.1:9202"
         ],
         "publish_address": "127.0.0.1:9202",
         "max_content_length_in_bytes": 104857600
@@ -81,19 +94,22 @@
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
       "version": "2.0.0",
-      "build": "de54438",
+      "build_type": "tar",
-      "http_address": "127.0.0.1:9203",
+      "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0",
+      "roles": [
+        "data",
+        "ingest"
+      ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "1",
-        "array.0": "d",
+        "shard_indexing_pressure_enabled": "true",
-        "array.1": "1",
+        "array": "[d, 1]"
-        "master": "false"
       },
       "http": {
         "bound_address": [
-          "127.0.0.1:9203",
+          "[::1]:9203",
-          "[::1]:9203"
+          "127.0.0.1:9203"
         ],
         "publish_address": "127.0.0.1:9203",
         "max_content_length_in_bytes": 104857600
@@ -105,19 +121,22 @@
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
       "version": "2.0.0",
-      "build": "de54438",
+      "build_type": "tar",
-      "http_address": "127.0.0.1:9204",
+      "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0",
+      "roles": [
+        "data",
+        "ingest"
+      ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "2",
-        "array.0": "d",
+        "shard_indexing_pressure_enabled": "true",
-        "array.1": "2",
+        "array": "[d, 2]"
-        "master": "false"
       },
       "http": {
         "bound_address": [
-          "127.0.0.1:9204",
+          "[::1]:9204",
-          "[::1]:9204"
+          "127.0.0.1:9204"
         ],
         "publish_address": "127.0.0.1:9204",
         "max_content_length_in_bytes": 104857600
@@ -129,19 +148,22 @@
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
       "version": "2.0.0",
-      "build": "de54438",
+      "build_type": "tar",
-      "http_address": "127.0.0.1:9205",
+      "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0",
+      "roles": [
+        "data",
+        "ingest"
+      ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "3",
-        "array.0": "d",
+        "shard_indexing_pressure_enabled": "true",
-        "array.1": "3",
+        "array": "[d, 3]"
-        "master": "false"
       },
       "http": {
         "bound_address": [
-          "127.0.0.1:9205",
+          "[::1]:9205",
-          "[::1]:9205"
+          "127.0.0.1:9205"
         ],
         "publish_address": "127.0.0.1:9205",
         "max_content_length_in_bytes": 104857600
@@ -153,20 +175,21 @@
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
       "version": "2.0.0",
-      "build": "de54438",
+      "build_type": "tar",
-      "http_address": "127.0.0.1:9206",
+      "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0",
+      "roles": [
+        "ingest"
+      ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "1",
-        "array.0": "c",
+        "shard_indexing_pressure_enabled": "true",
-        "data": "false",
+        "array": "[c, 1]"
-        "array.1": "1",
-        "master": "false"
       },
       "http": {
         "bound_address": [
-          "127.0.0.1:9206",
+          "[::1]:9206",
-          "[::1]:9206"
+          "127.0.0.1:9206"
         ],
         "publish_address": "127.0.0.1:9206",
         "max_content_length_in_bytes": 104857600
@@ -178,20 +201,21 @@
       "host": "127.0.0.1",
       "ip": "127.0.0.1",
       "version": "2.0.0",
-      "build": "de54438",
+      "build_type": "tar",
-      "http_address": "127.0.0.1:9207",
+      "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0",
+      "roles": [
+        "ingest"
+      ],
       "attributes": {
         "dummy": "everyone_has_me",
         "number": "2",
-        "array.0": "c",
+        "shard_indexing_pressure_enabled": "true",
-        "data": "false",
+        "array": "[c, 2]"
-        "array.1": "2",
-        "master": "false"
       },
       "http": {
         "bound_address": [
-          "127.0.0.1:9207",
+          "[::1]:9207",
-          "[::1]:9207"
+          "127.0.0.1:9207"
         ],
         "publish_address": "127.0.0.1:9207",
         "max_content_length_in_bytes": 104857600

client/sniffer/src/test/resources/5.0.0_nodes_http.json

@@ -1,217 +0,0 @@
-{
-  "_nodes": {
-    "total": 8,
-    "successful": 8,
-    "failed": 0
-  },
-  "cluster_name": "opensearch",
-  "nodes": {
-    "0S4r3NurTYSFSb8R9SxwWA": {
-      "name": "m1",
-      "transport_address": "127.0.0.1:9300",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "5.0.0",
-      "build_hash": "253032b",
-      "roles": [
-        "master",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "1",
-        "array.0": "m",
-        "array.1": "1"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9200",
-          "127.0.0.1:9200"
-        ],
-        "publish_address": "127.0.0.1:9200",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "k_CBrMXARkS57Qb5-3Mw5g": {
-      "name": "m2",
-      "transport_address": "127.0.0.1:9301",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "5.0.0",
-      "build_hash": "253032b",
-      "roles": [
-        "master",
-        "data",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "2",
-        "array.0": "m",
-        "array.1": "2"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9201",
-          "127.0.0.1:9201"
-        ],
-        "publish_address": "127.0.0.1:9201",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "6eynRPQ1RleJTeGDuTR9mw": {
-      "name": "m3",
-      "transport_address": "127.0.0.1:9302",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "5.0.0",
-      "build_hash": "253032b",
-      "roles": [
-        "master",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "3",
-        "array.0": "m",
-        "array.1": "3"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9202",
-          "127.0.0.1:9202"
-        ],
-        "publish_address": "127.0.0.1:9202",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "cbGC-ay1QNWaESvEh5513w": {
-      "name": "d1",
-      "transport_address": "127.0.0.1:9303",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "5.0.0",
-      "build_hash": "253032b",
-      "roles": [
-        "data",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "1",
-        "array.0": "d",
-        "array.1": "1"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9203",
-          "127.0.0.1:9203"
-        ],
-        "publish_address": "127.0.0.1:9203",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "LexndPpXR2ytYsU5fTElnQ": {
-      "name": "d2",
-      "transport_address": "127.0.0.1:9304",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "5.0.0",
-      "build_hash": "253032b",
-      "roles": [
-        "data",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "2",
-        "array.0": "d",
-        "array.1": "2"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9204",
-          "127.0.0.1:9204"
-        ],
-        "publish_address": "127.0.0.1:9204",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "SbNG1DKYSBu20zfOz2gDZQ": {
-      "name": "d3",
-      "transport_address": "127.0.0.1:9305",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "5.0.0",
-      "build_hash": "253032b",
-      "roles": [
-        "data",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "3",
-        "array.0": "d",
-        "array.1": "3"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9205",
-          "127.0.0.1:9205"
-        ],
-        "publish_address": "127.0.0.1:9205",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "fM4H-m2WTDWmsGsL7jIJew": {
-      "name": "c1",
-      "transport_address": "127.0.0.1:9306",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "5.0.0",
-      "build_hash": "253032b",
-      "roles": [
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "1",
-        "array.0": "c",
-        "array.1": "1"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9206",
-          "127.0.0.1:9206"
-        ],
-        "publish_address": "127.0.0.1:9206",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "pFoh7d0BTbqqI3HKd9na5A": {
-      "name": "c2",
-      "transport_address": "127.0.0.1:9307",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "5.0.0",
-      "build_hash": "253032b",
-      "roles": [
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "2",
-        "array.0": "c",
-        "array.1": "2"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9207",
-          "127.0.0.1:9207"
-        ],
-        "publish_address": "127.0.0.1:9207",
-        "max_content_length_in_bytes": 104857600
-      }
-    }
-  }
-}

client/sniffer/src/test/resources/6.0.0_nodes_http.json

@@ -1,217 +0,0 @@
-{
-  "_nodes": {
-    "total": 8,
-    "successful": 8,
-    "failed": 0
-  },
-  "cluster_name": "opensearch",
-  "nodes": {
-    "ikXK_skVTfWkhONhldnbkw": {
-      "name": "m1",
-      "transport_address": "127.0.0.1:9300",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "6.0.0",
-      "build_hash": "8f0685b",
-      "roles": [
-        "master",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "1",
-        "array.0": "m",
-        "array.1": "1"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9200",
-          "127.0.0.1:9200"
-        ],
-        "publish_address": "127.0.0.1:9200",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "TMHa34w4RqeuYoHCfJGXZg": {
-      "name": "m2",
-      "transport_address": "127.0.0.1:9301",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "6.0.0",
-      "build_hash": "8f0685b",
-      "roles": [
-        "master",
-        "data",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "2",
-        "array.0": "m",
-        "array.1": "2"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9201",
-          "127.0.0.1:9201"
-        ],
-        "publish_address": "127.0.0.1:9201",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "lzaMRJTVT166sgVZdQ5thA": {
-      "name": "m3",
-      "transport_address": "127.0.0.1:9302",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "6.0.0",
-      "build_hash": "8f0685b",
-      "roles": [
-        "master",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "3",
-        "array.0": "m",
-        "array.1": "3"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9202",
-          "127.0.0.1:9202"
-        ],
-        "publish_address": "127.0.0.1:9202",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "tGP5sUecSd6BLTWk1NWF8Q": {
-      "name": "d1",
-      "transport_address": "127.0.0.1:9303",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "6.0.0",
-      "build_hash": "8f0685b",
-      "roles": [
-        "data",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "1",
-        "array.0": "d",
-        "array.1": "1"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9203",
-          "127.0.0.1:9203"
-        ],
-        "publish_address": "127.0.0.1:9203",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "c1UgW5ROTkSa2YnM_T56tw": {
-      "name": "d2",
-      "transport_address": "127.0.0.1:9304",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "6.0.0",
-      "build_hash": "8f0685b",
-      "roles": [
-        "data",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "2",
-        "array.0": "d",
-        "array.1": "2"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9204",
-          "127.0.0.1:9204"
-        ],
-        "publish_address": "127.0.0.1:9204",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "QM9yjqjmS72MstpNYV_trg": {
-      "name": "d3",
-      "transport_address": "127.0.0.1:9305",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "6.0.0",
-      "build_hash": "8f0685b",
-      "roles": [
-        "data",
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "3",
-        "array.0": "d",
-        "array.1": "3"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9205",
-          "127.0.0.1:9205"
-        ],
-        "publish_address": "127.0.0.1:9205",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "wLtzAssoQYeX_4TstgCj0Q": {
-      "name": "c1",
-      "transport_address": "127.0.0.1:9306",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "6.0.0",
-      "build_hash": "8f0685b",
-      "roles": [
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "1",
-        "array.0": "c",
-        "array.1": "1"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9206",
-          "127.0.0.1:9206"
-        ],
-        "publish_address": "127.0.0.1:9206",
-        "max_content_length_in_bytes": 104857600
-      }
-    },
-    "ONOzpst8TH-ZebG7fxGwaA": {
-      "name": "c2",
-      "transport_address": "127.0.0.1:9307",
-      "host": "127.0.0.1",
-      "ip": "127.0.0.1",
-      "version": "6.0.0",
-      "build_hash": "8f0685b",
-      "roles": [
-        "ingest"
-      ],
-      "attributes": {
-        "dummy": "everyone_has_me",
-        "number": "2",
-        "array.0": "c",
-        "array.1": "2"
-      },
-      "http": {
-        "bound_address": [
-          "[::1]:9207",
-          "127.0.0.1:9207"
-        ],
-        "publish_address": "127.0.0.1:9207",
-        "max_content_length_in_bytes": 104857600
-      }
-    }
-  }
-}

client/sniffer/src/test/resources/create_test_nodes_info.bash

@@ -21,15 +21,11 @@ work=$(mktemp -d)
 pushd ${work} >> /dev/null
 echo Working in ${work}
 
-wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.0.0/elasticsearch-2.0.0.tar.gz
+wget https://artifacts.opensearch.org/releases/core/opensearch/1.0.0/opensearch-min-1.0.0-linux-x64.tar.gz
-wget https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-5.0.0.tar.gz
+wget https://artifacts.opensearch.org/releases/core/opensearch/2.0.0/opensearch-min-2.0.0-linux-x64.tar.gz
-wget https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-6.0.0.tar.gz
-sha1sum -c - << __SHAs
-e369d8579bd3a2e8b5344278d5043f19f14cac88 elasticsearch-2.0.0.tar.gz
-d25f6547bccec9f0b5ea7583815f96a6f50849e0 elasticsearch-5.0.0.tar.gz
-__SHAs
 sha512sum -c - << __SHAs
-25bb622d2fc557d8b8eded634a9b333766f7b58e701359e1bcfafee390776eb323cb7ea7a5e02e8803e25d8b1d3aabec0ec1b0cf492d0bab5689686fe440181c elasticsearch-6.0.0.tar.gz
+96595cd3b173188d8a3f0f18d7bfa2457782839d06b519f01a99b4dc0280f81b08ba1d01bd1aef454feaa574cbbd04d3ad9a1f6a829182627e914f3e58f2899f opensearch-min-1.0.0-linux-x64.tar.gz
+5b91456a2eb517bc48f13bec0a3f9c220494bd5fe979946dce6cfc3fa7ca00b003927157194d62f2a1c36c850eda74c70b93fbffa91bb082b2e1a17985d50976 opensearch-min-2.0.0-linux-x64.tar.gz
 __SHAs
 
 
@@ -40,37 +36,38 @@ function do_version() {
     mkdir -p ${version}
     pushd ${version} >> /dev/null
 
-    tar xf ../opensearch-${version}.tar.gz
+    tar xf ../opensearch-min-${version}-linux-x64.tar.gz
     local http_port=9200
     for node in ${nodes}; do
         mkdir ${node}
         cp -r opensearch-${version}/* ${node}
-        local master=$([[ "$node" =~ ^m.* ]] && echo true || echo false)
+        local cluster_manager=$([[ "$node" =~ ^m.* ]] && echo 'cluster_manager,' || echo '')
-        local data=$([[ "$node" =~ ^d.* ]] && echo true || echo false)
+        # 'cluster_manager' role is add in version 2.x and above, use 'master' role in 1.x
-        # m2 is always master and data for these test just so we have a node like that
+        cluster_manager=$([[ ! "$cluster_manager" == '' && ${version} =~ ^1\. ]] && echo 'master,' || echo ${cluster_manager})
-        data=$([[ "$node" == 'm2' ]] && echo true || echo ${data})
+        local data=$([[ "$node" =~ ^d.* ]] && echo 'data,' || echo '')
-        local attr=$([ ${version} == '2.0.0' ] && echo '' || echo '.attr')
+        # m2 is always cluster_manager and data for these test just so we have a node like that
+        data=$([[ "$node" == 'm2' ]] && echo 'data,' || echo ${data})
+        # setting name 'cluster.initial_cluster_manager_nodes' is add in version 2.x and above
+        local initial_cluster_manager_nodes=$([[ ${version} =~ ^1\. ]] && echo 'initial_master_nodes' || echo 'initial_cluster_manager_nodes')
         local transport_port=$((http_port+100))
 
-        cat >> ${node}/config/opensearch.yml << __ES_YML
+        cat >> ${node}/config/opensearch.yml << __OPENSEARCH_YML
 node.name:          ${node}
-node.master:        ${master}
+node.roles:         [${cluster_manager} ${data} ingest]
-node.data:          ${data}
+node.attr.dummy:    everyone_has_me
-node${attr}.dummy:  everyone_has_me
+node.attr.number:   ${node:1}
-node${attr}.number: ${node:1}
+node.attr.array:    [${node:0:1}, ${node:1}]
-node${attr}.array:  [${node:0:1}, ${node:1}]
 http.port:          ${http_port}
 transport.tcp.port: ${transport_port}
-discovery.zen.minimum_master_nodes: 3
+cluster.${initial_cluster_manager_nodes}: [m1, m2, m3]
-discovery.zen.ping.unicast.hosts: ['localhost:9300','localhost:9301','localhost:9302']
+discovery.seed_hosts: ['localhost:9300','localhost:9301','localhost:9302']
-__ES_YML
+__OPENSEARCH_YML
 
-        if [ ${version} != '2.0.0' ]; then
+        # configure the JVM heap size
         perl -pi -e 's/-Xm([sx]).+/-Xm${1}512m/g' ${node}/config/jvm.options
-        fi
 
         echo "starting ${version}/${node}..."
-        ${node}/bin/opensearch -d -p ${node}/pidfile
+        ${node}/bin/opensearch -d -p pidfile
 
         ((http_port++))
     done
@@ -99,9 +96,8 @@ __ES_YML
     popd >> /dev/null
 }
 
-JAVA_HOME=$JAVA8_HOME do_version 2.0.0
+JAVA_HOME=$JAVA11_HOME do_version 1.0.0
-JAVA_HOME=$JAVA8_HOME do_version 5.0.0
+JAVA_HOME=$JAVA11_HOME do_version 2.0.0
-JAVA_HOME=$JAVA8_HOME do_version 6.0.0
 
 popd >> /dev/null
 rm -rf ${work}