From fce52fe6bb320e6c8fd4e6c1073569d686a0ffb3 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Mon, 27 Jul 2020 10:08:13 +0100 Subject: [PATCH] =?UTF-8?q?HBASE-24758=20Avoid=20flooding=20replication=20?= =?UTF-8?q?source=20RSes=20logs=20when=20no=20sinks=E2=80=A6=20(#2118)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Josh Elser Signed-off-by: Viraj Jasani (cherry picked from commit 8c0d7fa5b8971de8bc7062675ba96a6091263776) --- .../hbase/replication/HBaseReplicationEndpoint.java | 4 ++-- .../HBaseInterClusterReplicationEndpoint.java | 12 ++++++++++-- .../regionserver/ReplicationSinkManager.java | 3 +++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java index 1ca70ad85dd..3cde0d5113a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/HBaseReplicationEndpoint.java @@ -168,8 +168,8 @@ public abstract class HBaseReplicationEndpoint extends BaseReplicationEndpoint } /** - * Get a list of all the addresses of all the region servers - * for this peer cluster + * Get a list of all the addresses of all the available region servers + * for this peer cluster, or an empty list if no region servers available at peer cluster. * @return list of addresses */ // Synchronize peer cluster connection attempts to avoid races and rate diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java index 9539d30eddf..4fb61466804 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java @@ -128,6 +128,8 @@ public class HBaseInterClusterReplicationEndpoint extends HBaseReplicationEndpoi private boolean dropOnDeletedTables; private boolean dropOnDeletedColumnFamilies; private boolean isSerial = false; + //Initialising as 0 to guarantee at least one logging message + private long lastSinkFetchTime = 0; /* * Some implementations of HBaseInterClusterReplicationEndpoint may require instantiating @@ -518,8 +520,14 @@ public class HBaseInterClusterReplicationEndpoint extends HBaseReplicationEndpoi int numSinks = replicationSinkMgr.getNumSinks(); if (numSinks == 0) { - LOG.warn("{} No replication sinks found, returning without replicating. " - + "The source should retry with the same set of edits.", logPeerId()); + if((System.currentTimeMillis() - lastSinkFetchTime) >= (maxRetriesMultiplier*1000)) { + LOG.warn( + "No replication sinks found, returning without replicating. " + + "The source should retry with the same set of edits. Not logging this again for " + + "the next {} seconds.", maxRetriesMultiplier); + lastSinkFetchTime = System.currentTimeMillis(); + } + sleepForRetries("No sinks available at peer", sleepMultiplier); return false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java index 3cd7884185d..4dc5b334f27 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSinkManager.java @@ -160,6 +160,9 @@ public class ReplicationSinkManager { */ public synchronized void chooseSinks() { List slaveAddresses = endpoint.getRegionServers(); + if(slaveAddresses.isEmpty()){ + LOG.warn("No sinks available at peer. Will not be able to replicate"); + } Collections.shuffle(slaveAddresses, random); int numSinks = (int) Math.ceil(slaveAddresses.size() * ratio); sinks = slaveAddresses.subList(0, numSinks);