HBASE-1660: script to handle rolling restarts

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@985084 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -835,6 +835,8 @@ Release 0.21.0 - Unreleased
    HBASE-2870  Add Backup CLI Option to HMaster (Nicolas Spiegelberg via Stack)
    HBASE-2868  Do some small cleanups in org.apache.hadoop.hbase.regionserver.wal
                (Alex Newman via Stack)
+   HBASE-1660  HBASE-1660 script to handle rolling restarts
+               (Nicolas Spiegelberg via Stack)
 
   NEW FEATURES
    HBASE-1961  HBase EC2 scripts

bin/hbase-daemon.sh

@@ -33,7 +33,7 @@
 # Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh
 
 usage="Usage: hbase-daemon.sh [--config <conf-dir>]\
- (start|stop) <hbase-command> \
+ (start|stop|restart) <hbase-command> \
  <args...>"
 
 # if no args specified, show usage
@@ -71,6 +71,24 @@ hbase_rotate_log ()
     fi
 }
 
+wait_until_done ()
+{
+    p=$1
+    cnt=${HBASE_SLAVE_TIMEOUT:-60}
+    origcnt=$cnt
+    while kill -0 $p > /dev/null 2>&1; do
+      if [ $cnt -gt 1 ]; then
+        cnt=`expr $cnt - 1`
+        sleep 1
+      else
+        echo "Process did not complete after $origcnt seconds, killing."
+        kill -9 $p
+        exit 1
+      fi
+    done
+    return 0
+}
+
 # get log directory
 if [ "$HBASE_LOG_DIR" = "" ]; then
   export HBASE_LOG_DIR="$HBASE_HOME/logs"
@@ -156,6 +174,22 @@ case $startStop in
     fi
     ;;
 
+  (restart)
+    thiscmd=$0
+    args=$@
+    # stop the command
+    $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args &
+    wait_until_done $!
+    # wait a user-specified sleep period
+    sp=${HBASE_SLAVE_SLEEP:-3}
+    if [ $sp -gt 0 ]; then
+      sleep $sp
+    fi
+    # start the command
+    $thiscmd --config "${HBASE_CONF_DIR}" start $command $args &
+    wait_until_done $!
+    ;;
+
   (*)
     echo $usage
     exit 1

bin/regionservers.sh

@@ -60,11 +60,16 @@ if [ "$HOSTLIST" = "" ]; then
 fi
 
 for regionserver in `cat "$HOSTLIST"`; do
- ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \
-   2>&1 | sed "s/^/$regionserver: /" &
- if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
-   sleep $HBASE_SLAVE_SLEEP
- fi
+  if ${HBASE_SLAVE_PARALLEL:-true}; then 
+    ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \
+      2>&1 | sed "s/^/$regionserver: /" &
+  else # run each command serially 
+    ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \
+      2>&1 | sed "s/^/$regionserver: /"
+  fi
+  if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
+    sleep $HBASE_SLAVE_SLEEP
+  fi
 done
 
 wait

bin/rolling-restart.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Copyright 2010 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+#
+# Run a shell command on all regionserver hosts.
+#
+# Environment Variables
+#
+#   HBASE_REGIONSERVERS    File naming remote hosts.
+#     Default is ${HADOOP_CONF_DIR}/regionservers
+#   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
+#   HBASE_CONF_DIR  Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
+#   HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+#   HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command.
+#   HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
+#
+# Modelled after $HADOOP_HOME/bin/slaves.sh.
+
+usage="Usage: $0 [--config <hbase-confdir>] commands..."
+
+bin=`dirname "$0"`
+bin=`cd "$bin">/dev/null; pwd`
+
+. "$bin"/hbase-config.sh
+
+# start hbase daemons
+errCode=$?
+if [ $errCode -ne 0 ]
+then
+  exit $errCode
+fi
+
+# quick function to get a value from the HBase config file
+distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
+if [ "$distMode" == 'false' ]; then
+  "$bin"/hbase-daemon.sh restart master
+else 
+  # stop all masters before re-start to avoid races for master znode
+  "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master 
+  "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
+    --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup
+
+  # make sure the master znode has been deleted before continuing
+  zparent=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool zookeeper.znode.parent`
+  if [ "$zparent" == "null" ]; then zparent="/hbase"; fi
+  zmaster=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool zookeeper.znode.master`
+  if [ "$zmaster" == "null" ]; then zmaster="master"; fi
+  zmaster=$zparent/$zmaster
+  echo -n "Waiting for Master ZNode to expire"
+  while bin/hbase zkcli stat $zmaster >/dev/null 2>&1; do
+    echo -n "."
+    sleep 1
+  done
+  echo #force a newline
+
+  # all masters are down, now restart
+  "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master
+  "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
+    --hosts "${HBASE_BACKUP_MASTERS}" start master-backup
+
+  # unlike the masters, roll all regionservers one-at-a-time
+  export HBASE_SLAVE_PARALLEL=false
+  "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
+    --hosts "${HBASE_REGIONSERVERS}" restart regionserver
+
+fi
+