diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java index fc42f18f9ee..c9513969b0c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; @@ -46,7 +47,7 @@ import org.slf4j.LoggerFactory; * * @since solr 7.5 */ -public final class ZookeeperStatusHandler extends RequestHandlerBase { +public class ZookeeperStatusHandler extends RequestHandlerBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final int ZOOKEEPER_DEFAULT_PORT = 2181; @@ -101,6 +102,7 @@ public final class ZookeeperStatusHandler extends RequestHandlerBase { Map stat = monitorZookeeper(zk); if (stat.containsKey("errors")) { errors.addAll((List)stat.get("errors")); + stat.remove("errors"); } details.add(stat); if ("true".equals(String.valueOf(stat.get("ok")))) { @@ -116,7 +118,7 @@ public final class ZookeeperStatusHandler extends RequestHandlerBase { standalone++; } } catch (SolrException se) { - log.warn("Failed talking to zookeeper" + zk, se); + log.warn("Failed talking to zookeeper " + zk, se); errors.add(se.getMessage()); Map stat = new HashMap<>(); stat.put("host", zk); @@ -180,31 +182,34 @@ public final class ZookeeperStatusHandler extends RequestHandlerBase { return zkStatus; } - private Map monitorZookeeper(String zkHostPort) throws SolrException { + protected Map monitorZookeeper(String zkHostPort) throws SolrException { Map obj = new HashMap<>(); List errors = new ArrayList<>(); obj.put("host", zkHostPort); List lines = getZkRawResponse(zkHostPort, "ruok"); + validateZkRawResponse(lines, zkHostPort,"ruok"); boolean ok = "imok".equals(lines.get(0)); obj.put("ok", ok); lines = getZkRawResponse(zkHostPort, "mntr"); + validateZkRawResponse(lines, zkHostPort,"mntr"); for (String line : lines) { String[] parts = line.split("\t"); if (parts.length >= 2) { obj.put(parts[0], parts[1]); } else { - String err = String.format("Unexpected line in 'mntr' response from Zookeeper %s: %s", zkHostPort, line); + String err = String.format(Locale.ENGLISH, "Unexpected line in 'mntr' response from Zookeeper %s: %s", zkHostPort, line); log.warn(err); errors.add(err); } } lines = getZkRawResponse(zkHostPort, "conf"); + validateZkRawResponse(lines, zkHostPort,"conf"); for (String line : lines) { String[] parts = line.split("="); if (parts.length >= 2) { obj.put(parts[0], parts[1]); } else if (!line.startsWith("membership:")) { - String err = String.format("Unexpected line in 'conf' response from Zookeeper %s: %s", zkHostPort, line); + String err = String.format(Locale.ENGLISH, "Unexpected line in 'conf' response from Zookeeper %s: %s", zkHostPort, line); log.warn(err); errors.add(err); } @@ -219,7 +224,7 @@ public final class ZookeeperStatusHandler extends RequestHandlerBase { * @param fourLetterWordCommand the custom 4-letter command to send to Zookeeper * @return a list of lines returned from Zookeeper */ - private List getZkRawResponse(String zkHostPort, String fourLetterWordCommand) { + protected List getZkRawResponse(String zkHostPort, String fourLetterWordCommand) { String[] hostPort = zkHostPort.split(":"); String host = hostPort[0]; int port = ZOOKEEPER_DEFAULT_PORT; @@ -235,18 +240,30 @@ public final class ZookeeperStatusHandler extends RequestHandlerBase { out.println(fourLetterWordCommand); List response = in.lines().collect(Collectors.toList()); log.debug("Got response from ZK on host {} and port {}: {}", host, port, response); - if (response == null || response.isEmpty()) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Empty response from Zookeeper " + zkHostPort); - } - if (response.size() == 1 && response.get(0).contains("not in the whitelist")) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Could not execute " + fourLetterWordCommand + - " towards ZK host " + zkHostPort + ". Add this line to the 'zoo.cfg' " + - "configuration file on each zookeeper node: '4lw.commands.whitelist=mntr,conf,ruok'. See also chapter " + - "'Setting Up an External ZooKeeper Ensemble' in the Solr Reference Guide."); - } return response; } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed talking to Zookeeper " + zkHostPort, e); } } + + /** + * Takes the raw response lines returned by {@link #getZkRawResponse(String, String)} and runs some validations + * @param response the lines + * @param zkHostPort the host + * @param fourLetterWordCommand the 4lw command + * @return true if validation succeeds + * @throws SolrException if validation fails + */ + protected boolean validateZkRawResponse(List response, String zkHostPort, String fourLetterWordCommand) { + if (response == null || response.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Empty response from Zookeeper " + zkHostPort); + } + if (response.size() == 1 && response.get(0).contains("not in the whitelist")) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Could not execute " + fourLetterWordCommand + + " towards ZK host " + zkHostPort + ". Add this line to the 'zoo.cfg' " + + "configuration file on each zookeeper node: '4lw.commands.whitelist=mntr,conf,ruok'. See also chapter " + + "'Setting Up an External ZooKeeper Ensemble' in the Solr Reference Guide."); + } + return true; + } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java index 4f29d8a7f1f..f33d7ffaece 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java @@ -20,6 +20,7 @@ package org.apache.solr.handler.admin; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URL; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -34,13 +35,22 @@ import org.apache.solr.client.solrj.response.DelegationTokenResponse; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SuppressForbidden; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.Answers; +import org.mockito.ArgumentMatchers; +import org.noggit.JSONUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class ZookeeperStatusHandlerTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -87,4 +97,56 @@ public class ZookeeperStatusHandlerTest extends SolrCloudTestCase { assertTrue(Integer.parseInt((String) details.get("zk_znode_count")) > 50); solr.close(); } + + @Test + public void testEnsembleStatusMock() { + assumeWorkingMockito(); + ZookeeperStatusHandler zkStatusHandler = mock(ZookeeperStatusHandler.class); + when(zkStatusHandler.getZkRawResponse("zoo1:2181", "ruok")).thenReturn(Arrays.asList("imok")); + when(zkStatusHandler.getZkRawResponse("zoo1:2181", "mntr")).thenReturn( + Arrays.asList("zk_version\t3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653, built on 05/03/2019 12:07 GMT", + "zk_avg_latency\t1")); + when(zkStatusHandler.getZkRawResponse("zoo1:2181", "conf")).thenReturn( + Arrays.asList("clientPort=2181", + "secureClientPort=-1", + "thisIsUnexpected", + "membership: ")); + + when(zkStatusHandler.getZkRawResponse("zoo2:2181", "ruok")).thenReturn(Arrays.asList("")); + + when(zkStatusHandler.getZkRawResponse("zoo3:2181", "ruok")).thenReturn(Arrays.asList("imok")); + when(zkStatusHandler.getZkRawResponse("zoo3:2181", "mntr")).thenReturn( + Arrays.asList("mntr is not executed because it is not in the whitelist.")); // Actual response from ZK if not whitelisted + when(zkStatusHandler.getZkRawResponse("zoo3:2181", "conf")).thenReturn( + Arrays.asList("clientPort=2181")); + + when(zkStatusHandler.getZkStatus(anyString())).thenCallRealMethod(); + when(zkStatusHandler.monitorZookeeper(anyString())).thenCallRealMethod(); + when(zkStatusHandler.validateZkRawResponse(ArgumentMatchers.any(), any(), any())).thenAnswer(Answers.CALLS_REAL_METHODS); + + Map mockStatus = zkStatusHandler.getZkStatus("zoo1:2181,zoo2:2181,zoo3:2181"); + String expected = "{\n" + + " \"ensembleSize\":3,\n" + + " \"details\":[\n" + + " {\n" + + " \"zk_version\":\"3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653, built on 05/03/2019 12:07 GMT\",\n" + + " \"zk_avg_latency\":\"1\",\n" + + " \"host\":\"zoo1:2181\",\n" + + " \"clientPort\":\"2181\",\n" + + " \"secureClientPort\":\"-1\",\n" + + " \"ok\":true},\n" + + " {\n" + + " \"host\":\"zoo2:2181\",\n" + + " \"ok\":false},\n" + + " {\n" + + " \"host\":\"zoo3:2181\",\n" + + " \"ok\":false}],\n" + + " \"zkHost\":\"zoo1:2181,zoo2:2181,zoo3:2181\",\n" + + " \"errors\":[\n" + + " \"Unexpected line in 'conf' response from Zookeeper zoo1:2181: thisIsUnexpected\",\n" + + " \"Empty response from Zookeeper zoo2:2181\",\n" + + " \"Could not execute mntr towards ZK host zoo3:2181. Add this line to the 'zoo.cfg' configuration file on each zookeeper node: '4lw.commands.whitelist=mntr,conf,ruok'. See also chapter 'Setting Up an External ZooKeeper Ensemble' in the Solr Reference Guide.\"],\n" + + " \"status\":\"yellow\"}"; + assertEquals(expected, JSONUtil.toJSON(mockStatus)); + } } \ No newline at end of file diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js index 296c75dd7bf..b0ba4217d78 100644 --- a/solr/webapp/web/js/angular/controllers/cloud.js +++ b/solr/webapp/web/js/angular/controllers/cloud.js @@ -530,7 +530,7 @@ var zkStatusSubController = function($scope, ZookeeperStatus) { $scope.initZookeeper = function() { ZookeeperStatus.monitor({}, function(data) { $scope.zkState = data.zkStatus; - $scope.mainKeys = ["ok", "clientPort", "zk_server_state", "zk_version", + $scope.mainKeys = ["ok", "clientPort", "secureClientPort", "zk_server_state", "zk_version", "zk_approximate_data_size", "zk_znode_count", "zk_num_alive_connections"]; $scope.detailKeys = ["dataDir", "dataLogDir", "zk_avg_latency", "zk_max_file_descriptor_count", "zk_watch_count", @@ -538,7 +538,14 @@ var zkStatusSubController = function($scope, ZookeeperStatus) { "tickTime", "maxClientCnxns", "minSessionTimeout", "maxSessionTimeout"]; $scope.ensembleMainKeys = ["serverId", "electionPort", "quorumPort"]; $scope.ensembleDetailKeys = ["peerType", "electionAlg", "initLimit", "syncLimit", - "zk_followers", "zk_synced_followers", "zk_pending_syncs"]; + "zk_followers", "zk_synced_followers", "zk_pending_syncs", + "server.1", "server.2", "server.3", "server.4", "server.5"]; + $scope.notEmptyRow = function(key) { + for (hostId in $scope.zkState.details) { + if (key in $scope.zkState.details[hostId]) return true; + } + return false; + }; }); }; diff --git a/solr/webapp/web/partials/cloud.html b/solr/webapp/web/partials/cloud.html index 1e64d2976f0..c2a878ad0d7 100644 --- a/solr/webapp/web/partials/cloud.html +++ b/solr/webapp/web/partials/cloud.html @@ -67,7 +67,7 @@ limitations under the License. {{host[key]}} - + {{key}} {{host[key]}}