HBASE-4785 Improve recovery time of HBase client when a region server dies.

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1205656 13f79535-47bb-0310-9956-ffa450edef68

src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java

@@ -23,7 +23,9 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.lang.reflect.Proxy;
 import java.lang.reflect.UndeclaredThrowableException;
+import java.net.ConnectException;
 import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -31,8 +33,8 @@ import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Map.Entry;
+import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.Callable;
@@ -512,6 +514,13 @@ public class HConnectionManager {
       cachedRegionLocations =
         new HashMap<Integer, SoftValueSortedMap<byte [], HRegionLocation>>();
 
+    // The presence of a server in the map implies it's likely that there is an
+    // entry in cachedRegionLocations that map to this server; but the absence
+    // of a server in this map guarentees that there is no entry in cache that
+    // maps to the absent server.
+    private final Set<String> cachedServers =
+        new HashSet<String>();
+
     // region cache prefetch is enabled by default. this set contains all
     // tables whose region cache prefetch are disabled.
     private final Set<Integer> regionCachePrefetchDisabledTables =
@@ -1116,6 +1125,35 @@ public class HConnectionManager {
       }
     }
 
+    /*
+     * Delete all cached entries of a table that maps to a specific location.
+     *
+     * @param tablename
+     * @param server
+     */
+    private void clearCachedLocationForServer(
+        final String server) {
+      boolean deletedSomething = false;
+      synchronized (this.cachedRegionLocations) {
+        if (!cachedServers.contains(server)) {
+          return;
+        }
+        for (SoftValueSortedMap<byte[], HRegionLocation> tableLocations :
+            cachedRegionLocations.values()) {
+          for (Entry<byte[], HRegionLocation> e : tableLocations.entrySet()) {
+            if (e.getValue().getServerAddress().toString().equals(server)) {
+              tableLocations.remove(e.getKey());
+              deletedSomething = true;
+            }
+          }
+        }
+        cachedServers.remove(server);
+      }
+      if (deletedSomething && LOG.isDebugEnabled()) {
+        LOG.debug("Removed all cached region locations that map to " + server);
+      }
+    }
+
     /*
      * @param tableName
      * @return Map of cached locations for passed <code>tableName</code>
@@ -1141,6 +1179,7 @@ public class HConnectionManager {
     public void clearRegionCache() {
       synchronized(this.cachedRegionLocations) {
         this.cachedRegionLocations.clear();
+        this.cachedServers.clear();
       }
     }
 
@@ -1159,7 +1198,12 @@ public class HConnectionManager {
       byte [] startKey = location.getRegionInfo().getStartKey();
       SoftValueSortedMap<byte [], HRegionLocation> tableLocations =
         getTableLocations(tableName);
-      if (tableLocations.put(startKey, location) == null) {
+      boolean hasNewCache = false;
+      synchronized (this.cachedRegionLocations) {
+        cachedServers.add(location.getServerAddress().toString());
+        hasNewCache = (tableLocations.put(startKey, location) == null);
+      }
+      if (hasNewCache) {
         LOG.debug("Cached location for " +
             location.getRegionInfo().getRegionNameAsString() +
             " is " + location.getHostnamePort());
@@ -1282,6 +1326,17 @@ public class HConnectionManager {
         } catch (Throwable t) {
           callable.shouldRetry(t);
           t = translateException(t);
+          if (t instanceof SocketTimeoutException ||
+              t instanceof ConnectException ||
+              t instanceof RetriesExhaustedException) {
+            // if thrown these exceptions, we clear all the cache entries that
+            // map to that slow/dead server; otherwise, let cache miss and ask
+            // .META. again to find the new location
+            HRegionLocation hrl = callable.location;
+            if (hrl != null) {
+              clearCachedLocationForServer(hrl.getServerAddress().toString());
+            }
+          }
           RetriesExhaustedException.ThrowableWithExtraContext qt =
             new RetriesExhaustedException.ThrowableWithExtraContext(t,
               System.currentTimeMillis(), callable.toString());

src/main/java/org/apache/hadoop/hbase/util/SoftValueSortedMap.java

@@ -24,6 +24,7 @@ import java.lang.ref.SoftReference;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
+import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
@@ -172,7 +173,13 @@ public class SoftValueSortedMap<K,V> implements SortedMap<K,V> {
   }
 
   public synchronized Set<Map.Entry<K,V>> entrySet() {
-    throw new RuntimeException("Not implemented");
+    checkReferences();
+    Set<Map.Entry<K, SoftValue<K, V>>> entries = this.internalMap.entrySet();
+    Set<Map.Entry<K, V>> realEntries = new LinkedHashSet<Map.Entry<K, V>>();
+    for (Map.Entry<K, SoftValue<K, V>> entry : entries) {
+      realEntries.add(entry.getValue());
+    }
+    return realEntries;
   }
 
   public synchronized Collection<V> values() {
@@ -185,12 +192,24 @@ public class SoftValueSortedMap<K,V> implements SortedMap<K,V> {
     return hardValues;
   }
 
-  private static class SoftValue<K,V> extends SoftReference<V> {
+  private static class SoftValue<K,V> extends SoftReference<V> implements Map.Entry<K, V> {
     final K key;
 
     SoftValue(K key, V value, ReferenceQueue q) {
       super(value, q);
       this.key = key;
     }
+
+    public K getKey() {
+      return this.key;
+    }
+
+    public V getValue() {
+      return get();
+    }
+
+    public V setValue(V value) {
+      throw new RuntimeException("Not implemented");
+    }
   }
 }