SOLR-5129: Timeout property for waiting ZK get started

solr/CHANGES.txt

@@ -151,6 +151,8 @@ Other Changes
 
 * SOLR-11249: Upgrade Jetty from 9.3.14.v20161028 to 9.3.20.v20170531 (Michael Braun via David Smiley)
 
+* SOLR-5129: Timeout property for waiting ZK get started. (Cao Manh Dat, Hrishikesh Gadre, Varun Thacker)
+
 ==================  7.0.0 ==================
 
 Versions of Major Components

solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java

@@ -277,7 +277,9 @@ public class SolrDispatchFilter extends BaseSolrFilter {
 
     String zkHost = System.getProperty("zkHost");
     if (!StringUtils.isEmpty(zkHost)) {
-      try (SolrZkClient zkClient = new SolrZkClient(zkHost, 30000)) {
+      int startUpZkTimeOut = Integer.getInteger("waitForZk", 30);
+      startUpZkTimeOut *= 1000;
+      try (SolrZkClient zkClient = new SolrZkClient(zkHost, startUpZkTimeOut)) {
         if (zkClient.exists("/solr.xml", true)) {
           log.info("solr.xml found in ZooKeeper. Loading...");
           byte[] data = zkClient.getData("/solr.xml", null, null, true);

solr/core/src/test/org/apache/solr/cloud/FailoverZKtest.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+public class FailoverZKtest extends SolrCloudTestCase {
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    System.setProperty("waitForZk", "60");
+    useFactory("solr.StandardDirectoryFactory");
+    configureCluster(2)
+        .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
+        .configure();
+  }
+
+  @AfterClass
+  public static void cleanUp() {
+    System.clearProperty("waitForZk");
+  }
+
+  public void testRestartZkWhenClusterDown() throws Exception {
+    String coll = "coll1";
+    CollectionAdminRequest.createCollection(coll, 2, 1).process(cluster.getSolrClient());
+    cluster.getSolrClient().add(coll, new SolrInputDocument("id", "1"));
+    for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
+      ChaosMonkey.stop(runner);
+    }
+    ZkTestServer zkTestServer = cluster.getZkServer();
+    zkTestServer.shutdown();
+    Thread[] threads = new Thread[cluster.getJettySolrRunners().size()];
+    for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+      final JettySolrRunner runner = cluster.getJettySolrRunner(i);
+      threads[i] = new Thread(() -> {
+        try {
+          ChaosMonkey.start(runner);
+        } catch (Exception e) {
+          e.printStackTrace();
+        }
+        });
+      threads[i].start();
+    }
+    Thread.sleep(5000);
+    zkTestServer = new ZkTestServer(zkTestServer.getZkDir(), zkTestServer.getPort());
+    zkTestServer.run();
+    for (Thread thread : threads) {
+      thread.join();
+    }
+    waitForLiveNodes(2);
+    waitForState("Timeout waiting for " + coll, coll, clusterShape(2, 1));
+    QueryResponse rsp = new QueryRequest(new SolrQuery("*:*")).process(cluster.getSolrClient(), coll);
+    assertEquals(1, rsp.getResults().getNumFound());
+    zkTestServer.shutdown();
+  }
+
+  private void waitForLiveNodes(int numNodes) throws InterruptedException, KeeperException {
+    ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
+    for (int i = 0; i < 100; i++) {
+      zkStateReader.updateLiveNodes();
+      if (zkStateReader.getClusterState().getLiveNodes().size() == numNodes) return;
+      Thread.sleep(200);
+    }
+    fail("Timeout waiting for number of live nodes = " + numNodes);
+  }
+}