HBASE-3052 Add ability to have multiple ZK servers in a quorum in MiniZooKeeperCluster for test writing

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1084008 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -86,6 +86,8 @@ Release 0.91.0 - Unreleased
                needs an upper bound added (Ted Yu via Stack)
    HBASE-3676  Update region server load for AssignmentManager through
                regionServerReport() (Ted Yu via Stack)
+   HBASE-3052  Add ability to have multiple ZK servers in a quorum in
+               MiniZooKeeperCluster for test writing (Liyin Tang via Stack)
 
   TASK
    HBASE-3559  Move report of split to master OFF the heartbeat channel

src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java

@@ -28,6 +28,8 @@ import java.io.Reader;
 import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -48,14 +50,21 @@ public class MiniZooKeeperCluster {
   private static final int CONNECTION_TIMEOUT = 30000;
 
   private boolean started;
-  private int clientPort = 21818; // use non-standard port
+
+  private int defaultClientPort = 21818; // use non-standard port
+  private int clientPort = defaultClientPort; 
+  private int zooKeeperCandidateNum = 0;
   
   private NIOServerCnxn.Factory standaloneServerFactory;
+  private int currentZooKeeper;
+  private List<ZooKeeperServer> zooKeeperServers;
   private int tickTime = 0;
 
   /** Create mini ZooKeeper cluster. */
   public MiniZooKeeperCluster() {
     this.started = false;
+    currentZooKeeper = -1;
+    zooKeeperServers = new ArrayList();
   }
 
   public void setClientPort(int clientPort) {
@@ -70,6 +79,10 @@ public class MiniZooKeeperCluster {
     this.tickTime = tickTime;
   }
   
+  public int getZooKeeperCandidateNum() {
+    return zooKeeperCandidateNum;
+  }
+
   // / XXX: From o.a.zk.t.ClientBase
   private static void setupTestEnv() {
     // during the tests we run with 100K prealloc in the logs.
@@ -80,29 +93,40 @@ public class MiniZooKeeperCluster {
     FileTxnLog.setPreallocSize(100);
   }
   
+  public int startup(File baseDir) throws IOException,
+  InterruptedException {
+    return startup(baseDir,1);
+  }
+
   /**
    * @param baseDir
+   * @param numZooKeeperServers
    * @return ClientPort server bound to.
    * @throws IOException
    * @throws InterruptedException
    */
-  public int startup(File baseDir) throws IOException,
+  public int startup(File baseDir, int numZooKeeperServers) throws IOException,
       InterruptedException {
+    if (numZooKeeperServers <= 0)
+      return -1;
 
     setupTestEnv();
 
     shutdown();
     
-    File dir = new File(baseDir, "zookeeper").getAbsoluteFile();
+    for (int i = 0; i < numZooKeeperServers; i++) {
-    recreateDir(dir);
+      File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile();
+      recreateDir(dir);
   
-    int tickTimeToUse;
+      int tickTimeToUse;
-    if (this.tickTime > 0) {
+      if (this.tickTime > 0) {
-      tickTimeToUse = this.tickTime;
+        tickTimeToUse = this.tickTime;
-    } else {
+      } else {
-      tickTimeToUse = TICK_TIME;
+        tickTimeToUse = TICK_TIME;
+      }
+      ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
+      zooKeeperServers.add(server);
     }
-    ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
     while (true) {
       try {
         standaloneServerFactory =
@@ -115,13 +139,14 @@ public class MiniZooKeeperCluster {
       }
       break;
     }
-    standaloneServerFactory.startup(server);
+    currentZooKeeper = 0;
-
+    standaloneServerFactory.startup(zooKeeperServers.get(currentZooKeeper));   
     if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) {
       throw new IOException("Waiting for startup of standalone server");
     }
 
     started = true;
+    zooKeeperCandidateNum = numZooKeeperServers-1;
     LOG.info("Started MiniZK Server on client port: " + clientPort);
     return clientPort;
   }
@@ -151,6 +176,56 @@ public class MiniZooKeeperCluster {
     }
 
     started = false;
+    zooKeeperCandidateNum = 0;
+    LOG.info("Shutdown MiniZK Server on client port: " + clientPort);
+  }
+  
+  /**@return clientPort return clientPort if there is another ZooKeeper Candidate can run; return
+   *         -1, if there is no candidates.
+   * @throws IOException
+   * @throws InterruptedException 
+   */
+  public int killCurrentZooKeeper() throws IOException, 
+                                        InterruptedException {
+    if (!started) {
+      return -1;
+    }
+    // Shutdown the current one
+    standaloneServerFactory.shutdown();
+    if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
+      throw new IOException("Waiting for shutdown of standalone server");
+    }
+    LOG.info("Kill the current MiniZK Server on client port: " 
+        + clientPort);
+    
+    if (zooKeeperCandidateNum == 0) {
+      return -1;
+    }
+    // Start another ZooKeeper Server
+    clientPort = defaultClientPort;
+    while (true) {
+      try {
+        standaloneServerFactory =
+          new NIOServerCnxn.Factory(new InetSocketAddress(clientPort));
+      } catch (BindException e) {
+        LOG.info("Failed binding ZK Server to client port: " + clientPort);
+        //this port is already in use. try to use another
+        clientPort++;
+        continue;
+      }
+      break;
+    }
+    currentZooKeeper = (++currentZooKeeper) % zooKeeperServers.size() ;
+    standaloneServerFactory.startup(zooKeeperServers.get(currentZooKeeper));
+
+    if (!waitForServerUp(clientPort, CONNECTION_TIMEOUT)) {
+      throw new IOException("Waiting for startup of standalone server");
+    }
+    
+    started = true;
+    zooKeeperCandidateNum--;
+    LOG.info("Started another candidate MiniZK Server on client port: " + clientPort);
+    return clientPort;
   }
 
   // XXX: From o.a.zk.t.ClientBase

src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java

@@ -245,18 +245,38 @@ public class HBaseTestingUtility {
    * @return zk cluster started.
    */
   public MiniZooKeeperCluster startMiniZKCluster() throws Exception {
-    return startMiniZKCluster(setupClusterTestBuildDir());
+    return startMiniZKCluster(setupClusterTestBuildDir(),1);
+
+  }
+  
+  /**
+   * Call this if you only want a zk cluster.
+   * @param zooKeeperServerNum
+   * @see #startMiniZKCluster() if you want zk + dfs + hbase mini cluster.
+   * @throws Exception
+   * @see #shutdownMiniZKCluster()
+   * @return zk cluster started.
+   */
+  public MiniZooKeeperCluster startMiniZKCluster(int zooKeeperServerNum) 
+      throws Exception {
+    return startMiniZKCluster(setupClusterTestBuildDir(), zooKeeperServerNum);
 
   }
   
   private MiniZooKeeperCluster startMiniZKCluster(final File dir)
+    throws Exception {
+    return startMiniZKCluster(dir,1);
+  }
+  
+  private MiniZooKeeperCluster startMiniZKCluster(final File dir, 
+      int zooKeeperServerNum)
   throws Exception {
     this.passedZkCluster = false;
     if (this.zkCluster != null) {
       throw new IOException("Cluster already running at " + dir);
     }
     this.zkCluster = new MiniZooKeeperCluster();
-    int clientPort = this.zkCluster.startup(dir);
+    int clientPort = this.zkCluster.startup(dir,zooKeeperServerNum);
     this.conf.set("hbase.zookeeper.property.clientPort",
       Integer.toString(clientPort));
     return this.zkCluster;

src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java

@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -137,6 +138,28 @@ public class TestHBaseTestingUtility {
     }
   }
   
+  @Test public void testMiniZooKeeper() throws Exception {
+    MiniZooKeeperCluster cluster1 = this.hbt.startMiniZKCluster();
+    try {
+      assertEquals(0, cluster1.getZooKeeperCandidateNum());
+      assertTrue((cluster1.killCurrentZooKeeper() == -1));
+    } finally {
+      cluster1.shutdown();
+    }
+    
+    this.hbt.shutdownMiniZKCluster();
+    
+    MiniZooKeeperCluster cluster2 = this.hbt.startMiniZKCluster(5);
+    try {
+      assertEquals(4, cluster2.getZooKeeperCandidateNum());
+      assertTrue((cluster2.killCurrentZooKeeper() > 0));
+      assertTrue((cluster2.killCurrentZooKeeper() > 0));
+      assertEquals(2, cluster2.getZooKeeperCandidateNum());
+    } finally {
+      cluster2.shutdown();
+    }
+  }
+
   @Test public void testMiniDFSCluster() throws Exception {
     MiniDFSCluster cluster = this.hbt.startMiniDFSCluster(1);
     FileSystem dfs = cluster.getFileSystem();