HBASE-21357 RS should abort if OOM in Reader thread

hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java

@@ -732,6 +732,17 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
             LOG.error(getName() + ": CancelledKeyException in Reader", e);
           } catch (IOException ex) {
             LOG.info(getName() + ": IOException in Reader", ex);
+          } catch (OutOfMemoryError e) {
+            if (getErrorHandler() != null) {
+              if (getErrorHandler().checkOOME(e)) {
+                RpcServer.LOG.info(Thread.currentThread().getName()
+                    + ": exiting on OutOfMemoryError");
+                return;
+              }
+            } else {
+              // rethrow if no handler
+              throw e;
+            }
           }
         }
       }