HBASE-511 Do exponential backoff in clients on NSRE, WRE, ISE, etc.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@680902 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -344,6 +344,7 @@ Release 0.2.0
                Cryans via JimK)
    HBASE-733   Enhance Cell so that it can contain multiple values at multiple
                timestamps
+   HBASE-511   Do exponential backoff in clients on NSRE, WRE, ISE, etc.
    
   OPTIMIZATIONS
    HBASE-430   Performance: Scanners and getRow return maps with duplicate data

conf/hbase-default.xml

@@ -112,11 +112,11 @@
   </property>
   <property>
     <name>hbase.client.retries.number</name>
-    <value>5</value>
+    <value>10</value>
     <description>Maximum retries.  Used as maximum for all retryable
     operations such as fetching of the root region from root region
     server, getting a cell's value, starting a row update, etc.
-    Default: 5.
+    Default: 10.
     </description>
   </property>
   <property>

src/java/org/apache/hadoop/hbase/HConstants.java

@@ -226,4 +226,10 @@ public interface HConstants {
   public static final String NAME = "NAME";
   public static final String VERSIONS = "VERSIONS";
   public static final String IN_MEMORY = "IN_MEMORY";
+  
+  /**
+   * This is a retry backoff multiplier table similar to the BSD TCP syn
+   * backoff table, a bit more aggressive than simple exponential backoff.
+   */ 
+  public static int RETRY_BACKOFF[] = { 1, 1, 1, 1, 2, 4, 8, 16, 32, 64 };
 }

src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java

@@ -126,6 +126,12 @@ public class HBaseAdmin {
     return this.connection.listTables();
   }
 
+  private long getPauseTime(int tries) {
+    if (tries >= HConstants.RETRY_BACKOFF.length)
+      tries = HConstants.RETRY_BACKOFF.length - 1;
+    return this.pause * HConstants.RETRY_BACKOFF[tries];
+  }
+
   /**
    * Creates a new table
    * 
@@ -155,7 +161,7 @@ public class HBaseAdmin {
         }
       }
       try {
-        Thread.sleep(pause);
+        Thread.sleep(getPauseTime(tries));
       } catch (InterruptedException e) {
         // continue
       }
@@ -274,7 +280,7 @@ public class HBaseAdmin {
       }
 
       try {
-        Thread.sleep(pause);
+        Thread.sleep(getPauseTime(tries));
       } catch (InterruptedException e) {
         // continue
       }
@@ -320,14 +326,15 @@ public class HBaseAdmin {
 
     // Wait until all regions are enabled
     
-    while (!isTableEnabled(tableName)) {
+    for (int tries = 0;
+        (tries < numRetries) && (!isTableEnabled(tableName));
+        tries++) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Sleep. Waiting for all regions to be enabled from " +
           Bytes.toString(tableName));
       }
       try {
-        Thread.sleep(pause);
+        Thread.sleep(getPauseTime(tries));
-        
       } catch (InterruptedException e) {
         // continue
       }
@@ -336,6 +343,9 @@ public class HBaseAdmin {
           Bytes.toString(tableName));
       }
     }
+    if (!isTableEnabled(tableName))
+      throw new IOException("unable to enable table " +
+        Bytes.toString(tableName));
     LOG.info("Enabled table " + Bytes.toString(tableName));
   }
 
@@ -379,13 +389,15 @@ public class HBaseAdmin {
     }
 
     // Wait until all regions are disabled
-    while (isTableEnabled(tableName)) {
+    for (int tries = 0;
+        (tries < numRetries) && (isTableEnabled(tableName));
+        tries++) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Sleep. Waiting for all regions to be disabled from " +
           Bytes.toString(tableName));
       }
       try {
-        Thread.sleep(pause);
+        Thread.sleep(getPauseTime(tries));
       } catch (InterruptedException e) {
         // continue
       }
@@ -394,6 +406,9 @@ public class HBaseAdmin {
           Bytes.toString(tableName));
       }
     }
+    if (isTableEnabled(tableName))
+      throw new IOException("unable to disable table " +
+        Bytes.toString(tableName));
     LOG.info("Disabled " + Bytes.toString(tableName));
   }
   

src/java/org/apache/hadoop/hbase/client/HConnectionManager.java

@@ -163,14 +163,20 @@ public class HConnectionManager implements HConstants {
             "Unable to find region server interface " + serverClassName, e);
       }
 
-      this.pause = conf.getLong("hbase.client.pause", 30 * 1000);
+      this.pause = conf.getLong("hbase.client.pause", 10 * 1000);
-      this.numRetries = conf.getInt("hbase.client.retries.number", 5);
+      this.numRetries = conf.getInt("hbase.client.retries.number", 10);
       this.maxRPCAttempts = conf.getInt("hbase.client.rpc.maxattempts", 1);
       
       this.master = null;
       this.masterChecked = false;
     }
 
+    private long getPauseTime(int tries) {
+      if (tries >= HConstants.RETRY_BACKOFF.length)
+        tries = HConstants.RETRY_BACKOFF.length - 1;
+      return this.pause * HConstants.RETRY_BACKOFF[tries];
+    }
+
     /** {@inheritDoc} */
     public HMasterInterface getMaster() throws MasterNotRunningException {
       HServerAddress masterLocation = null;
@@ -199,13 +205,14 @@ public class HConnectionManager implements HConstants {
               break;
             }
             LOG.info("Attempt " + tries + " of " + this.numRetries +
-              " failed with <" + e + ">. Retrying after sleep of " + this.pause);
+              " failed with <" + e + ">. Retrying after sleep of " +
+              getPauseTime(tries));
           }
 
           // We either cannot connect to master or it is not running. Sleep & retry
           
           try {
-            Thread.sleep(this.pause);
+            Thread.sleep(getPauseTime(tries));
           } catch (InterruptedException e) {
             // continue
           }
@@ -578,7 +585,7 @@ public class HConnectionManager implements HConstants {
         }
       
         try{
-          Thread.sleep(pause);              
+          Thread.sleep(getPauseTime(tries));              
         } catch (InterruptedException e){
           // continue
         }
@@ -795,7 +802,7 @@ public class HConnectionManager implements HConstants {
               if (LOG.isDebugEnabled()) {
                 LOG.debug("Sleeping. Waiting for root region.");
               }
-              Thread.sleep(pause);
+              Thread.sleep(getPauseTime(tries));
               if (LOG.isDebugEnabled()) {
                 LOG.debug("Wake. Retry finding root region.");
               }
@@ -837,7 +844,7 @@ public class HConnectionManager implements HConstants {
             if (LOG.isDebugEnabled()) {
               LOG.debug("Root region location changed. Sleeping.");
             }
-            Thread.sleep(pause);
+            Thread.sleep(getPauseTime(tries));
             if (LOG.isDebugEnabled()) {
               LOG.debug("Wake. Retry finding root region.");
             }
@@ -890,7 +897,7 @@ public class HConnectionManager implements HConstants {
           }
         }
         try {
-          Thread.sleep(pause);
+          Thread.sleep(getPauseTime(tries));
         } catch (InterruptedException e) {
           // continue
         }