From ab478ba5f33d45b98a3dd2b12038680d443cc1e1 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Mon, 6 Jun 2022 13:41:56 -0700 Subject: [PATCH] Fix the support of RestClient Node Sniffer for version 2.x and update tests (#3487) Fix the support of RestClient Node Sniffer for OpenSearch 2.x, and update unit tests for OpenSearch. The current code contains the logic to be compatible with Elasticsearch 2.x version, which is conflict with OpenSearch 2.x, so removed that part of legacy code. * Update the script create_test_nodes_info.bash to dump the response of Nodes Info API GET _nodes/http for OpenSearch 1.0 and 2.0 version, which used for unit test. * Remove the support of Elasticsearch version 2.x for the Sniffer * Update unit test to validate the Sniffer compatible with OpenSearch 1.x and 2.x * Update the API response parser to meet the array notation (in ES 6.1 and above) for the node attributes setting. It will result the value of `node.attr` setting will not be parsed as array in the Sniffer, when using the Sniffer on cluster in Elasticsearch 6.0 and above. * Replace "master" node role with "cluster_manager" in unit test Signed-off-by: Tianli Feng --- .../client/sniff/OpenSearchNodesSniffer.java | 70 +----- .../OpenSearchNodesSnifferParseTests.java | 96 +++----- .../sniff/OpenSearchNodesSnifferTests.java | 16 +- ..._nodes_http.json => 1.0.0_nodes_http.json} | 79 ++++--- .../src/test/resources/2.0.0_nodes_http.json | 144 +++++++----- .../src/test/resources/5.0.0_nodes_http.json | 217 ------------------ .../src/test/resources/6.0.0_nodes_http.json | 217 ------------------ .../resources/create_test_nodes_info.bash | 56 +++-- 8 files changed, 193 insertions(+), 702 deletions(-) rename client/sniffer/src/test/resources/{7.3.0_nodes_http.json => 1.0.0_nodes_http.json} (77%) delete mode 100644 client/sniffer/src/test/resources/5.0.0_nodes_http.json delete mode 100644 client/sniffer/src/test/resources/6.0.0_nodes_http.json diff --git a/client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java b/client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java index 2829439627d..c1a0fcf9a8a 100644 --- a/client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java +++ b/client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -241,74 +242,23 @@ public final class OpenSearchNodesSniffer implements NodesSniffer { } Map> realAttributes = new HashMap<>(protoAttributes.size()); - List keys = new ArrayList<>(protoAttributes.keySet()); - for (String key : keys) { - if (key.endsWith(".0")) { - String realKey = key.substring(0, key.length() - 2); - List values = new ArrayList<>(); - int i = 0; - while (true) { - String value = protoAttributes.remove(realKey + "." + i); - if (value == null) { - break; - } - values.add(value); - i++; - } - realAttributes.put(realKey, unmodifiableList(values)); - } - } for (Map.Entry entry : protoAttributes.entrySet()) { - realAttributes.put(entry.getKey(), singletonList(entry.getValue())); + if (entry.getValue().startsWith("[")) { + // Convert string array to list + String value = entry.getValue(); + String[] values = value.substring(1, value.length() - 1).split(", "); + realAttributes.put(entry.getKey(), unmodifiableList(Arrays.asList(values))); + } else { + realAttributes.put(entry.getKey(), singletonList(entry.getValue())); + } } - if (version.startsWith("2.")) { - /* - * 2.x doesn't send roles, instead we try to read them from - * attributes. - */ - boolean clientAttribute = v2RoleAttributeValue(realAttributes, "client", false); - Boolean masterAttribute = v2RoleAttributeValue(realAttributes, "master", null); - Boolean dataAttribute = v2RoleAttributeValue(realAttributes, "data", null); - if ((masterAttribute == null && false == clientAttribute) || masterAttribute) { - roles.add("master"); - } - if ((dataAttribute == null && false == clientAttribute) || dataAttribute) { - roles.add("data"); - } - } else { - assert sawRoles : "didn't see roles for [" + nodeId + "]"; - } + assert sawRoles : "didn't see roles for [" + nodeId + "]"; assert boundHosts.contains(publishedHost) : "[" + nodeId + "] doesn't make sense! publishedHost should be in boundHosts"; logger.trace("adding node [" + nodeId + "]"); return new Node(publishedHost, boundHosts, name, version, new Roles(roles), unmodifiableMap(realAttributes)); } - /** - * Returns {@code defaultValue} if the attribute didn't come back, - * {@code true} or {@code false} if it did come back as - * either of those, or throws an IOException if the attribute - * came back in a strange way. - */ - private static Boolean v2RoleAttributeValue(Map> attributes, String name, Boolean defaultValue) - throws IOException { - List valueList = attributes.remove(name); - if (valueList == null) { - return defaultValue; - } - if (valueList.size() != 1) { - throw new IOException("expected only a single attribute value for [" + name + "] but got " + valueList); - } - switch (valueList.get(0)) { - case "true": - return true; - case "false": - return false; - default: - throw new IOException("expected [" + name + "] to be either [true] or [false] but was [" + valueList.get(0) + "]"); - } - } - /** * The supported host schemes. */ diff --git a/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java b/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java index a9ff47eab53..58b60ac13de 100644 --- a/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java +++ b/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java @@ -45,8 +45,8 @@ import org.opensearch.client.sniff.OpenSearchNodesSniffer.Scheme; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -85,59 +85,31 @@ public class OpenSearchNodesSnifferParseTests extends RestClientTestCase { } } + public void test1x() throws IOException { + checkFile( + "1.0.0_nodes_http.json", + node(9200, "m1", "1.0.0", "master", "ingest"), + node(9201, "m2", "1.0.0", "master", "data", "ingest"), + node(9202, "m3", "1.0.0", "master", "ingest"), + node(9203, "d1", "1.0.0", "data", "ingest"), + node(9204, "d2", "1.0.0", "data", "ingest"), + node(9205, "d3", "1.0.0", "data", "ingest"), + node(9206, "c1", "1.0.0", "ingest"), + node(9207, "c2", "1.0.0", "ingest") + ); + } + public void test2x() throws IOException { checkFile( "2.0.0_nodes_http.json", - node(9200, "m1", "2.0.0", true, false, false), - node(9201, "m2", "2.0.0", true, true, false), - node(9202, "m3", "2.0.0", true, false, false), - node(9203, "d1", "2.0.0", false, true, false), - node(9204, "d2", "2.0.0", false, true, false), - node(9205, "d3", "2.0.0", false, true, false), - node(9206, "c1", "2.0.0", false, false, false), - node(9207, "c2", "2.0.0", false, false, false) - ); - } - - public void test5x() throws IOException { - checkFile( - "5.0.0_nodes_http.json", - node(9200, "m1", "5.0.0", true, false, true), - node(9201, "m2", "5.0.0", true, true, true), - node(9202, "m3", "5.0.0", true, false, true), - node(9203, "d1", "5.0.0", false, true, true), - node(9204, "d2", "5.0.0", false, true, true), - node(9205, "d3", "5.0.0", false, true, true), - node(9206, "c1", "5.0.0", false, false, true), - node(9207, "c2", "5.0.0", false, false, true) - ); - } - - public void test6x() throws IOException { - checkFile( - "6.0.0_nodes_http.json", - node(9200, "m1", "6.0.0", true, false, true), - node(9201, "m2", "6.0.0", true, true, true), - node(9202, "m3", "6.0.0", true, false, true), - node(9203, "d1", "6.0.0", false, true, true), - node(9204, "d2", "6.0.0", false, true, true), - node(9205, "d3", "6.0.0", false, true, true), - node(9206, "c1", "6.0.0", false, false, true), - node(9207, "c2", "6.0.0", false, false, true) - ); - } - - public void test7x() throws IOException { - checkFile( - "7.3.0_nodes_http.json", - node(9200, "m1", "7.3.0", "master", "ingest"), - node(9201, "m2", "7.3.0", "master", "data", "ingest"), - node(9202, "m3", "7.3.0", "master", "ingest"), - node(9203, "d1", "7.3.0", "data", "ingest", "ml"), - node(9204, "d2", "7.3.0", "data", "ingest"), - node(9205, "d3", "7.3.0", "data", "ingest"), - node(9206, "c1", "7.3.0", "ingest"), - node(9207, "c2", "7.3.0", "ingest") + node(9200, "m1", "2.0.0", "cluster_manager", "ingest"), + node(9201, "m2", "2.0.0", "cluster_manager", "data", "ingest"), + node(9202, "m3", "2.0.0", "cluster_manager", "ingest"), + node(9203, "d1", "2.0.0", "data", "ingest"), + node(9204, "d2", "2.0.0", "data", "ingest"), + node(9205, "d3", "2.0.0", "data", "ingest"), + node(9206, "c1", "2.0.0", "ingest"), + node(9207, "c2", "2.0.0", "ingest") ); } @@ -163,20 +135,6 @@ public class OpenSearchNodesSnifferParseTests extends RestClientTestCase { assertEquals("http", nodes.get(0).getHost().getSchemeName()); } - private Node node(int port, String name, String version, boolean master, boolean data, boolean ingest) { - final Set roles = new TreeSet<>(); - if (master) { - roles.add("master"); - } - if (data) { - roles.add("data"); - } - if (ingest) { - roles.add("ingest"); - } - return node(port, name, version, roles); - } - private Node node(int port, String name, String version, String... roles) { return node(port, name, version, new TreeSet<>(Arrays.asList(roles))); } @@ -184,11 +142,15 @@ public class OpenSearchNodesSnifferParseTests extends RestClientTestCase { private Node node(int port, String name, String version, Set roles) { HttpHost host = new HttpHost("127.0.0.1", port); Set boundHosts = new HashSet<>(2); - boundHosts.add(host); boundHosts.add(new HttpHost("[::1]", port)); - Map> attributes = new HashMap<>(); + boundHosts.add(host); + Map> attributes = new LinkedHashMap<>(); // LinkedHashMap to preserve insertion order attributes.put("dummy", singletonList("everyone_has_me")); attributes.put("number", singletonList(name.substring(1))); + if (!version.startsWith("1.0") && !version.startsWith("1.1")) { + // Shard Indexing Pressure feature is added in version 1.2.0 + attributes.put("shard_indexing_pressure_enabled", singletonList(Boolean.TRUE.toString())); + } attributes.put("array", Arrays.asList(name.substring(0, 1), name.substring(1))); return new Node(host, boundHosts, name, version, new Roles(new TreeSet<>(roles)), attributes); } diff --git a/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java b/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java index 993844524c2..8cc6f5f0068 100644 --- a/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java +++ b/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java @@ -234,7 +234,7 @@ public class OpenSearchNodesSnifferTests extends RestClientTestCase { final Set nodeRoles = new TreeSet<>(); if (randomBoolean()) { - nodeRoles.add("master"); + nodeRoles.add("cluster_manager"); } if (randomBoolean()) { nodeRoles.add("data"); @@ -283,12 +283,12 @@ public class OpenSearchNodesSnifferTests extends RestClientTestCase { generator.writeEndObject(); } - List roles = Arrays.asList(new String[] { "master", "data", "ingest" }); + List roles = Arrays.asList(new String[] { "cluster_manager", "data", "ingest" }); Collections.shuffle(roles, getRandom()); generator.writeArrayFieldStart("roles"); for (String role : roles) { - if ("master".equals(role) && node.getRoles().isMasterEligible()) { - generator.writeString("master"); + if ("cluster_manager".equals(role) && node.getRoles().isMasterEligible()) { + generator.writeString("cluster_manager"); } if ("data".equals(role) && node.getRoles().isData()) { generator.writeString("data"); @@ -307,13 +307,7 @@ public class OpenSearchNodesSnifferTests extends RestClientTestCase { if (numAttributes > 0) { generator.writeObjectFieldStart("attributes"); for (Map.Entry> entry : attributes.entrySet()) { - if (entry.getValue().size() == 1) { - generator.writeStringField(entry.getKey(), entry.getValue().get(0)); - } else { - for (int v = 0; v < entry.getValue().size(); v++) { - generator.writeStringField(entry.getKey() + "." + v, entry.getValue().get(v)); - } - } + generator.writeStringField(entry.getKey(), entry.getValue().toString()); } generator.writeEndObject(); } diff --git a/client/sniffer/src/test/resources/7.3.0_nodes_http.json b/client/sniffer/src/test/resources/1.0.0_nodes_http.json similarity index 77% rename from client/sniffer/src/test/resources/7.3.0_nodes_http.json rename to client/sniffer/src/test/resources/1.0.0_nodes_http.json index 9e85511fadb..5557f0c7955 100644 --- a/client/sniffer/src/test/resources/7.3.0_nodes_http.json +++ b/client/sniffer/src/test/resources/1.0.0_nodes_http.json @@ -11,17 +11,17 @@ "transport_address": "127.0.0.1:9300", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ - "master", - "ingest" + "ingest", + "master" ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "m", - "array.1": "1" + "array": "[m, 1]" }, "http": { "bound_address": [ @@ -37,18 +37,18 @@ "transport_address": "127.0.0.1:9301", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ - "master", "data", - "ingest" + "ingest", + "master" ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "m", - "array.1": "2" + "array": "[m, 2]" }, "http": { "bound_address": [ @@ -64,17 +64,17 @@ "transport_address": "127.0.0.1:9302", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ - "master", - "ingest" + "ingest", + "master" ], "attributes": { "dummy": "everyone_has_me", "number": "3", - "array.0": "m", - "array.1": "3" + "array": "[m, 3]" }, "http": { "bound_address": [ @@ -90,18 +90,17 @@ "transport_address": "127.0.0.1:9303", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "data", - "ingest", - "ml" + "ingest" ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "d", - "array.1": "1" + "array": "[d, 1]" }, "http": { "bound_address": [ @@ -117,8 +116,9 @@ "transport_address": "127.0.0.1:9304", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "data", "ingest" @@ -126,8 +126,7 @@ "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "d", - "array.1": "2" + "array": "[d, 2]" }, "http": { "bound_address": [ @@ -143,8 +142,9 @@ "transport_address": "127.0.0.1:9305", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "data", "ingest" @@ -152,8 +152,7 @@ "attributes": { "dummy": "everyone_has_me", "number": "3", - "array.0": "d", - "array.1": "3" + "array": "[d, 3]" }, "http": { "bound_address": [ @@ -169,16 +168,16 @@ "transport_address": "127.0.0.1:9306", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "ingest" ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "c", - "array.1": "1" + "array": "[c, 1]" }, "http": { "bound_address": [ @@ -194,16 +193,16 @@ "transport_address": "127.0.0.1:9307", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "ingest" ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "c", - "array.1": "2" + "array": "[c, 2]" }, "http": { "bound_address": [ diff --git a/client/sniffer/src/test/resources/2.0.0_nodes_http.json b/client/sniffer/src/test/resources/2.0.0_nodes_http.json index 4e8dbbcba58..e1b75d460d7 100644 --- a/client/sniffer/src/test/resources/2.0.0_nodes_http.json +++ b/client/sniffer/src/test/resources/2.0.0_nodes_http.json @@ -1,4 +1,9 @@ { + "_nodes": { + "total": 8, + "successful": 8, + "failed": 0 + }, "cluster_name": "opensearch", "nodes": { "qr-SOrELSaGW8SlU8nflBw": { @@ -7,20 +12,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9200", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "cluster_manager", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "m", - "data": "false", - "array.1": "1", - "master": "true" + "array": "[m, 1]", + "shard_indexing_pressure_enabled": "true" }, "http": { "bound_address": [ - "127.0.0.1:9200", - "[::1]:9200" + "[::1]:9200", + "127.0.0.1:9200" ], "publish_address": "127.0.0.1:9200", "max_content_length_in_bytes": 104857600 @@ -32,19 +39,23 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9201", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "cluster_manager", + "data", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "m", - "array.1": "2", - "master": "true" + "shard_indexing_pressure_enabled": "true", + "array": "[m, 2]" }, "http": { "bound_address": [ - "127.0.0.1:9201", - "[::1]:9201" + "[::1]:9201", + "127.0.0.1:9201" ], "publish_address": "127.0.0.1:9201", "max_content_length_in_bytes": 104857600 @@ -56,20 +67,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9202", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "cluster_manager", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "3", - "array.0": "m", - "data": "false", - "array.1": "3", - "master": "true" + "shard_indexing_pressure_enabled": "true", + "array": "[m, 3]" }, "http": { "bound_address": [ - "127.0.0.1:9202", - "[::1]:9202" + "[::1]:9202", + "127.0.0.1:9202" ], "publish_address": "127.0.0.1:9202", "max_content_length_in_bytes": 104857600 @@ -81,19 +94,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9203", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "data", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "d", - "array.1": "1", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[d, 1]" }, "http": { "bound_address": [ - "127.0.0.1:9203", - "[::1]:9203" + "[::1]:9203", + "127.0.0.1:9203" ], "publish_address": "127.0.0.1:9203", "max_content_length_in_bytes": 104857600 @@ -105,19 +121,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9204", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "data", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "d", - "array.1": "2", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[d, 2]" }, "http": { "bound_address": [ - "127.0.0.1:9204", - "[::1]:9204" + "[::1]:9204", + "127.0.0.1:9204" ], "publish_address": "127.0.0.1:9204", "max_content_length_in_bytes": 104857600 @@ -129,19 +148,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9205", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "data", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "3", - "array.0": "d", - "array.1": "3", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[d, 3]" }, "http": { "bound_address": [ - "127.0.0.1:9205", - "[::1]:9205" + "[::1]:9205", + "127.0.0.1:9205" ], "publish_address": "127.0.0.1:9205", "max_content_length_in_bytes": 104857600 @@ -153,20 +175,21 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9206", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "c", - "data": "false", - "array.1": "1", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[c, 1]" }, "http": { "bound_address": [ - "127.0.0.1:9206", - "[::1]:9206" + "[::1]:9206", + "127.0.0.1:9206" ], "publish_address": "127.0.0.1:9206", "max_content_length_in_bytes": 104857600 @@ -178,20 +201,21 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9207", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "c", - "data": "false", - "array.1": "2", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[c, 2]" }, "http": { "bound_address": [ - "127.0.0.1:9207", - "[::1]:9207" + "[::1]:9207", + "127.0.0.1:9207" ], "publish_address": "127.0.0.1:9207", "max_content_length_in_bytes": 104857600 diff --git a/client/sniffer/src/test/resources/5.0.0_nodes_http.json b/client/sniffer/src/test/resources/5.0.0_nodes_http.json deleted file mode 100644 index 4eb0443bc09..00000000000 --- a/client/sniffer/src/test/resources/5.0.0_nodes_http.json +++ /dev/null @@ -1,217 +0,0 @@ -{ - "_nodes": { - "total": 8, - "successful": 8, - "failed": 0 - }, - "cluster_name": "opensearch", - "nodes": { - "0S4r3NurTYSFSb8R9SxwWA": { - "name": "m1", - "transport_address": "127.0.0.1:9300", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "master", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "m", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9200", - "127.0.0.1:9200" - ], - "publish_address": "127.0.0.1:9200", - "max_content_length_in_bytes": 104857600 - } - }, - "k_CBrMXARkS57Qb5-3Mw5g": { - "name": "m2", - "transport_address": "127.0.0.1:9301", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "master", - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "m", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9201", - "127.0.0.1:9201" - ], - "publish_address": "127.0.0.1:9201", - "max_content_length_in_bytes": 104857600 - } - }, - "6eynRPQ1RleJTeGDuTR9mw": { - "name": "m3", - "transport_address": "127.0.0.1:9302", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "master", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "3", - "array.0": "m", - "array.1": "3" - }, - "http": { - "bound_address": [ - "[::1]:9202", - "127.0.0.1:9202" - ], - "publish_address": "127.0.0.1:9202", - "max_content_length_in_bytes": 104857600 - } - }, - "cbGC-ay1QNWaESvEh5513w": { - "name": "d1", - "transport_address": "127.0.0.1:9303", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "d", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9203", - "127.0.0.1:9203" - ], - "publish_address": "127.0.0.1:9203", - "max_content_length_in_bytes": 104857600 - } - }, - "LexndPpXR2ytYsU5fTElnQ": { - "name": "d2", - "transport_address": "127.0.0.1:9304", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "d", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9204", - "127.0.0.1:9204" - ], - "publish_address": "127.0.0.1:9204", - "max_content_length_in_bytes": 104857600 - } - }, - "SbNG1DKYSBu20zfOz2gDZQ": { - "name": "d3", - "transport_address": "127.0.0.1:9305", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "3", - "array.0": "d", - "array.1": "3" - }, - "http": { - "bound_address": [ - "[::1]:9205", - "127.0.0.1:9205" - ], - "publish_address": "127.0.0.1:9205", - "max_content_length_in_bytes": 104857600 - } - }, - "fM4H-m2WTDWmsGsL7jIJew": { - "name": "c1", - "transport_address": "127.0.0.1:9306", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "c", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9206", - "127.0.0.1:9206" - ], - "publish_address": "127.0.0.1:9206", - "max_content_length_in_bytes": 104857600 - } - }, - "pFoh7d0BTbqqI3HKd9na5A": { - "name": "c2", - "transport_address": "127.0.0.1:9307", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "c", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9207", - "127.0.0.1:9207" - ], - "publish_address": "127.0.0.1:9207", - "max_content_length_in_bytes": 104857600 - } - } - } -} diff --git a/client/sniffer/src/test/resources/6.0.0_nodes_http.json b/client/sniffer/src/test/resources/6.0.0_nodes_http.json deleted file mode 100644 index adc8f535d6a..00000000000 --- a/client/sniffer/src/test/resources/6.0.0_nodes_http.json +++ /dev/null @@ -1,217 +0,0 @@ -{ - "_nodes": { - "total": 8, - "successful": 8, - "failed": 0 - }, - "cluster_name": "opensearch", - "nodes": { - "ikXK_skVTfWkhONhldnbkw": { - "name": "m1", - "transport_address": "127.0.0.1:9300", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "master", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "m", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9200", - "127.0.0.1:9200" - ], - "publish_address": "127.0.0.1:9200", - "max_content_length_in_bytes": 104857600 - } - }, - "TMHa34w4RqeuYoHCfJGXZg": { - "name": "m2", - "transport_address": "127.0.0.1:9301", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "master", - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "m", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9201", - "127.0.0.1:9201" - ], - "publish_address": "127.0.0.1:9201", - "max_content_length_in_bytes": 104857600 - } - }, - "lzaMRJTVT166sgVZdQ5thA": { - "name": "m3", - "transport_address": "127.0.0.1:9302", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "master", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "3", - "array.0": "m", - "array.1": "3" - }, - "http": { - "bound_address": [ - "[::1]:9202", - "127.0.0.1:9202" - ], - "publish_address": "127.0.0.1:9202", - "max_content_length_in_bytes": 104857600 - } - }, - "tGP5sUecSd6BLTWk1NWF8Q": { - "name": "d1", - "transport_address": "127.0.0.1:9303", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "d", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9203", - "127.0.0.1:9203" - ], - "publish_address": "127.0.0.1:9203", - "max_content_length_in_bytes": 104857600 - } - }, - "c1UgW5ROTkSa2YnM_T56tw": { - "name": "d2", - "transport_address": "127.0.0.1:9304", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "d", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9204", - "127.0.0.1:9204" - ], - "publish_address": "127.0.0.1:9204", - "max_content_length_in_bytes": 104857600 - } - }, - "QM9yjqjmS72MstpNYV_trg": { - "name": "d3", - "transport_address": "127.0.0.1:9305", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "3", - "array.0": "d", - "array.1": "3" - }, - "http": { - "bound_address": [ - "[::1]:9205", - "127.0.0.1:9205" - ], - "publish_address": "127.0.0.1:9205", - "max_content_length_in_bytes": 104857600 - } - }, - "wLtzAssoQYeX_4TstgCj0Q": { - "name": "c1", - "transport_address": "127.0.0.1:9306", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "c", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9206", - "127.0.0.1:9206" - ], - "publish_address": "127.0.0.1:9206", - "max_content_length_in_bytes": 104857600 - } - }, - "ONOzpst8TH-ZebG7fxGwaA": { - "name": "c2", - "transport_address": "127.0.0.1:9307", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "c", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9207", - "127.0.0.1:9207" - ], - "publish_address": "127.0.0.1:9207", - "max_content_length_in_bytes": 104857600 - } - } - } -} diff --git a/client/sniffer/src/test/resources/create_test_nodes_info.bash b/client/sniffer/src/test/resources/create_test_nodes_info.bash index 06350be4ba2..78e67562d81 100644 --- a/client/sniffer/src/test/resources/create_test_nodes_info.bash +++ b/client/sniffer/src/test/resources/create_test_nodes_info.bash @@ -21,15 +21,11 @@ work=$(mktemp -d) pushd ${work} >> /dev/null echo Working in ${work} -wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.0.0/elasticsearch-2.0.0.tar.gz -wget https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-5.0.0.tar.gz -wget https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-6.0.0.tar.gz -sha1sum -c - << __SHAs -e369d8579bd3a2e8b5344278d5043f19f14cac88 elasticsearch-2.0.0.tar.gz -d25f6547bccec9f0b5ea7583815f96a6f50849e0 elasticsearch-5.0.0.tar.gz -__SHAs +wget https://artifacts.opensearch.org/releases/core/opensearch/1.0.0/opensearch-min-1.0.0-linux-x64.tar.gz +wget https://artifacts.opensearch.org/releases/core/opensearch/2.0.0/opensearch-min-2.0.0-linux-x64.tar.gz sha512sum -c - << __SHAs -25bb622d2fc557d8b8eded634a9b333766f7b58e701359e1bcfafee390776eb323cb7ea7a5e02e8803e25d8b1d3aabec0ec1b0cf492d0bab5689686fe440181c elasticsearch-6.0.0.tar.gz +96595cd3b173188d8a3f0f18d7bfa2457782839d06b519f01a99b4dc0280f81b08ba1d01bd1aef454feaa574cbbd04d3ad9a1f6a829182627e914f3e58f2899f opensearch-min-1.0.0-linux-x64.tar.gz +5b91456a2eb517bc48f13bec0a3f9c220494bd5fe979946dce6cfc3fa7ca00b003927157194d62f2a1c36c850eda74c70b93fbffa91bb082b2e1a17985d50976 opensearch-min-2.0.0-linux-x64.tar.gz __SHAs @@ -40,37 +36,38 @@ function do_version() { mkdir -p ${version} pushd ${version} >> /dev/null - tar xf ../opensearch-${version}.tar.gz + tar xf ../opensearch-min-${version}-linux-x64.tar.gz local http_port=9200 for node in ${nodes}; do mkdir ${node} cp -r opensearch-${version}/* ${node} - local master=$([[ "$node" =~ ^m.* ]] && echo true || echo false) - local data=$([[ "$node" =~ ^d.* ]] && echo true || echo false) - # m2 is always master and data for these test just so we have a node like that - data=$([[ "$node" == 'm2' ]] && echo true || echo ${data}) - local attr=$([ ${version} == '2.0.0' ] && echo '' || echo '.attr') + local cluster_manager=$([[ "$node" =~ ^m.* ]] && echo 'cluster_manager,' || echo '') + # 'cluster_manager' role is add in version 2.x and above, use 'master' role in 1.x + cluster_manager=$([[ ! "$cluster_manager" == '' && ${version} =~ ^1\. ]] && echo 'master,' || echo ${cluster_manager}) + local data=$([[ "$node" =~ ^d.* ]] && echo 'data,' || echo '') + # m2 is always cluster_manager and data for these test just so we have a node like that + data=$([[ "$node" == 'm2' ]] && echo 'data,' || echo ${data}) + # setting name 'cluster.initial_cluster_manager_nodes' is add in version 2.x and above + local initial_cluster_manager_nodes=$([[ ${version} =~ ^1\. ]] && echo 'initial_master_nodes' || echo 'initial_cluster_manager_nodes') local transport_port=$((http_port+100)) - cat >> ${node}/config/opensearch.yml << __ES_YML + cat >> ${node}/config/opensearch.yml << __OPENSEARCH_YML node.name: ${node} -node.master: ${master} -node.data: ${data} -node${attr}.dummy: everyone_has_me -node${attr}.number: ${node:1} -node${attr}.array: [${node:0:1}, ${node:1}] +node.roles: [${cluster_manager} ${data} ingest] +node.attr.dummy: everyone_has_me +node.attr.number: ${node:1} +node.attr.array: [${node:0:1}, ${node:1}] http.port: ${http_port} transport.tcp.port: ${transport_port} -discovery.zen.minimum_master_nodes: 3 -discovery.zen.ping.unicast.hosts: ['localhost:9300','localhost:9301','localhost:9302'] -__ES_YML +cluster.${initial_cluster_manager_nodes}: [m1, m2, m3] +discovery.seed_hosts: ['localhost:9300','localhost:9301','localhost:9302'] +__OPENSEARCH_YML - if [ ${version} != '2.0.0' ]; then - perl -pi -e 's/-Xm([sx]).+/-Xm${1}512m/g' ${node}/config/jvm.options - fi + # configure the JVM heap size + perl -pi -e 's/-Xm([sx]).+/-Xm${1}512m/g' ${node}/config/jvm.options echo "starting ${version}/${node}..." - ${node}/bin/opensearch -d -p ${node}/pidfile + ${node}/bin/opensearch -d -p pidfile ((http_port++)) done @@ -99,9 +96,8 @@ __ES_YML popd >> /dev/null } -JAVA_HOME=$JAVA8_HOME do_version 2.0.0 -JAVA_HOME=$JAVA8_HOME do_version 5.0.0 -JAVA_HOME=$JAVA8_HOME do_version 6.0.0 +JAVA_HOME=$JAVA11_HOME do_version 1.0.0 +JAVA_HOME=$JAVA11_HOME do_version 2.0.0 popd >> /dev/null rm -rf ${work}