From c4f7a53747140b0b2a940c28f071c82d930ad3a3 Mon Sep 17 00:00:00 2001 From: Jean-Daniel Cryans Date: Fri, 31 Aug 2012 04:59:02 +0000 Subject: [PATCH] HBASE-6321 ReplicationSource dies reading the peer's id HBASE-6647 [performance regression] appendNoSync/HBASE-4528 doesn't take deferred log flush into account git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1379289 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/hbase/regionserver/HRegion.java | 31 ++++++---- .../hadoop/hbase/regionserver/wal/HLog.java | 13 +++- .../replication/ReplicationZookeeper.java | 61 +++++++++++++++---- .../regionserver/ReplicationSource.java | 20 +++--- 4 files changed, 90 insertions(+), 35 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 42ebd70e4f2..15abe87a27f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -2295,10 +2295,8 @@ public class HRegion implements HeapSize { // , Writable{ // ------------------------- // STEP 7. Sync wal. // ------------------------- - if (walEdit.size() > 0 && - (this.regionInfo.isMetaRegion() || - !this.htableDescriptor.isDeferredLogFlush())) { - this.log.sync(txid); + if (walEdit.size() > 0) { + syncOrDefer(txid); } walSyncSuccessful = true; // ------------------------------------------------------------------ @@ -4498,10 +4496,8 @@ public class HRegion implements HeapSize { // , Writable{ acquiredLocks = null; } // 10. Sync edit log - if (txid != 0 && - (this.regionInfo.isMetaRegion() || - !this.htableDescriptor.isDeferredLogFlush())) { - this.log.sync(txid); + if (txid != 0) { + syncOrDefer(txid); } walSyncSuccessful = true; } @@ -4750,7 +4746,7 @@ public class HRegion implements HeapSize { // , Writable{ releaseRowLock(lid); } if (writeToWAL) { - this.log.sync(txid); // sync the transaction log outside the rowlock + syncOrDefer(txid); // sync the transaction log outside the rowlock } } finally { closeRegionOperation(); @@ -4878,7 +4874,7 @@ public class HRegion implements HeapSize { // , Writable{ releaseRowLock(lid); } if (writeToWAL) { - this.log.sync(txid); // sync the transaction log outside the rowlock + syncOrDefer(txid); // sync the transaction log outside the rowlock } } finally { closeRegionOperation(); @@ -4976,7 +4972,7 @@ public class HRegion implements HeapSize { // , Writable{ releaseRowLock(lid); } if (writeToWAL) { - this.log.sync(txid); // sync the transaction log outside the rowlock + syncOrDefer(txid); // sync the transaction log outside the rowlock } } finally { closeRegionOperation(); @@ -5389,6 +5385,19 @@ public class HRegion implements HeapSize { // , Writable{ dataInMemoryWithoutWAL.addAndGet(putSize); } + /** + * Calls sync with the given transaction ID if the region's table is not + * deferring it. + * @param txid should sync up to which transaction + * @throws IOException If anything goes wrong with DFS + */ + private void syncOrDefer(long txid) throws IOException { + if (this.regionInfo.isMetaRegion() || + !this.htableDescriptor.isDeferredLogFlush()) { + this.log.sync(txid); + } + } + /** * A mocked list implementaion - discards all updates. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java index 0edb81f1e7a..2ce83f4a2fa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java @@ -1193,10 +1193,14 @@ public class HLog implements Syncable { } /** - * This thread is responsible to call syncFs and buffer up the writers while - * it happens. + * This class is responsible to hold the HLog's appended Entry list + * and to sync them according to a configurable interval. + * + * Deferred log flushing works first by piggy backing on this process by + * simply not sync'ing the appended Entry. It can also be sync'd by other + * non-deferred log flushed entries outside of this thread. */ - class LogSyncer extends HasThread { + class LogSyncer extends HasThread { private final long optionalFlushInterval; @@ -1227,6 +1231,9 @@ public class HLog implements Syncable { closeLogSyncer.wait(this.optionalFlushInterval); } } + // Calling sync since we waited or had unflushed entries. + // Entries appended but not sync'd are taken care of here AKA + // deferred log flush sync(); } catch (IOException e) { LOG.error("Error while syncing, requesting close of hlog ", e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java index 76650d3860d..14b832fa9ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationZookeeper.java @@ -30,6 +30,7 @@ import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; +import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; @@ -43,7 +44,9 @@ import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.hadoop.hbase.replication.regionserver.Replication; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker; @@ -253,19 +256,7 @@ public class ReplicationZookeeper implements Closeable{ try { addresses = fetchSlavesAddresses(peer.getZkw()); } catch (KeeperException ke) { - if (ke instanceof ConnectionLossException - || ke instanceof SessionExpiredException) { - LOG.warn( - "Lost the ZooKeeper connection for peer " + peer.getClusterKey(), - ke); - try { - peer.reloadZkWatcher(); - } catch(IOException io) { - LOG.warn( - "Creation of ZookeeperWatcher failed for peer " - + peer.getClusterKey(), io); - } - } + reconnectPeer(ke, peer); addresses = Collections.emptyList(); } peer.setRegionServers(addresses); @@ -976,6 +967,50 @@ public class ReplicationZookeeper implements Closeable{ return 0; } + /** + * Returns the UUID of the provided peer id. Should a connection loss or session + * expiration happen, the ZK handler will be reopened once and if it still doesn't + * work then it will bail and return null. + * @param peerId the peer's ID that will be converted into a UUID + * @return a UUID or null if there's a ZK connection issue + */ + public UUID getPeerUUID(String peerId) { + ReplicationPeer peer = getPeerClusters().get(peerId); + UUID peerUUID = null; + try { + peerUUID = getUUIDForCluster(peer.getZkw()); + } catch (KeeperException ke) { + reconnectPeer(ke, peer); + } + return peerUUID; + } + + /** + * Get the UUID for the provided ZK watcher. Doesn't handle any ZK exceptions + * @param zkw watcher connected to an ensemble + * @return the UUID read from zookeeper + * @throws KeeperException + */ + public UUID getUUIDForCluster(ZooKeeperWatcher zkw) throws KeeperException { + return UUID.fromString(ZKClusterId.readClusterIdZNode(zkw)); + } + + private void reconnectPeer(KeeperException ke, ReplicationPeer peer) { + if (ke instanceof ConnectionLossException + || ke instanceof SessionExpiredException) { + LOG.warn( + "Lost the ZooKeeper connection for peer " + peer.getClusterKey(), + ke); + try { + peer.reloadZkWatcher(); + } catch(IOException io) { + LOG.warn( + "Creation of ZookeeperWatcher failed for peer " + + peer.getClusterKey(), io); + } + } + } + public void registerRegionServerListener(ZooKeeperListener listener) { this.zookeeper.registerListener(listener); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index 3d36b43b3d1..c9889c84f2b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -189,8 +189,7 @@ public class ReplicationSource extends Thread this.metrics = new ReplicationSourceMetrics(peerClusterZnode); try { - this.clusterId = UUID.fromString(ZKClusterId.readClusterIdZNode(zkHelper - .getZookeeperWatcher())); + this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher()); } catch (KeeperException ke) { throw new IOException("Could not read cluster id", ke); } @@ -250,13 +249,19 @@ public class ReplicationSource extends Thread metrics.clear(); return; } + int sleepMultiplier = 1; // delay this until we are in an asynchronous thread - try { - this.peerClusterId = UUID.fromString(ZKClusterId - .readClusterIdZNode(zkHelper.getPeerClusters().get(peerId).getZkw())); - } catch (KeeperException ke) { - this.terminate("Could not read peer's cluster id", ke); + while (this.peerClusterId == null) { + this.peerClusterId = zkHelper.getPeerUUID(this.peerId); + if (this.peerClusterId == null) { + if (sleepForRetries("Cannot contact the peer's zk ensemble", sleepMultiplier)) { + sleepMultiplier++; + } + } } + // resetting to 1 to reuse later + sleepMultiplier = 1; + LOG.info("Replicating "+clusterId + " -> " + peerClusterId); // If this is recovered, the queue is already full and the first log @@ -270,7 +275,6 @@ public class ReplicationSource extends Thread peerClusterZnode, e); } } - int sleepMultiplier = 1; // Loop until we close down while (isActive()) { // Sleep until replication is enabled again