From d2d6945e8951391a3b04d89f2879b51d7df26bb7 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Mon, 8 Dec 2014 10:12:27 -0600 Subject: [PATCH] HBASE-12655 WALPerformanceEvaluation should only add stat gathering listener once per WAL. --- .../java/org/apache/hadoop/hbase/wal/WAL.java | 3 + .../hbase/wal/TestDefaultWALProvider.java | 27 ++++++++ .../hbase/wal/WALPerformanceEvaluation.java | 61 +++++++++++-------- 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java index 787a34b0dfa..23f8c9fa6ef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java @@ -41,6 +41,9 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit; /** * A Write Ahead Log (WAL) provides service for reading, writing waledits. This interface provides * APIs for WAL users (such as RegionServer) to use the WAL (do append, sync, etc). + * + * Note that some internals, such as log rolling and performance evaluation tools, will use + * WAL.equals to determine if they have already seen a given WAL. */ @InterfaceAudience.Private @InterfaceStability.Evolving diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java index 8eef74c97a1..28cd84956ab 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java @@ -19,9 +19,14 @@ package org.apache.hadoop.hbase.wal; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; @@ -329,4 +334,26 @@ public class TestDefaultWALProvider { new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"}); assertEquals(0, errCode); } + + /** + * Ensure that we can use Set.add to deduplicate WALs + */ + @Test + public void setMembershipDedups() throws IOException { + final Configuration localConf = new Configuration(conf); + localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName()); + final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName()); + try { + final Set seen = new HashSet(1); + final Random random = new Random(); + assertTrue("first attempt to add WAL from default provider should work.", + seen.add(wals.getWAL(Bytes.toBytes(random.nextInt())))); + for (int i = 0; i < 1000; i++) { + assertFalse("default wal provider is only supposed to return a single wal, which should " + + "compare as .equals itself.", seen.add(wals.getWAL(Bytes.toBytes(random.nextInt())))); + } + } finally { + wals.close(); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java index c00ebcaa3e2..bfc7147286e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java @@ -21,9 +21,11 @@ package org.apache.hadoop.hbase.wal; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -460,43 +462,48 @@ public final class WALPerformanceEvaluation extends Configured implements Tool { System.exit(1); } + private final Set walsListenedTo = new HashSet(); + private HRegion openRegion(final FileSystem fs, final Path dir, final HTableDescriptor htd, final WALFactory wals, final long whenToRoll) throws IOException { // Initialize HRegion HRegionInfo regionInfo = new HRegionInfo(htd.getTableName()); // Initialize WAL final WAL wal = wals.getWAL(regionInfo.getEncodedNameAsBytes()); - wal.registerWALActionsListener(new WALActionsListener.Base() { - private int appends = 0; + // If we haven't already, attach a listener to this wal to handle rolls and metrics. + if (walsListenedTo.add(wal)) { + wal.registerWALActionsListener(new WALActionsListener.Base() { + private int appends = 0; - @Override - public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey, - WALEdit logEdit) { - this.appends++; - if (this.appends % whenToRoll == 0) { - LOG.info("Rolling after " + appends + " edits"); - // We used to do explicit call to rollWriter but changed it to a request - // to avoid dead lock (there are less threads going on in this class than - // in the regionserver -- regionserver does not have the issue). - // TODO I think this means no rolling actually happens; the request relies on there - // being a LogRoller. - DefaultWALProvider.requestLogRoll(wal); + @Override + public void visitLogEntryBeforeWrite(HTableDescriptor htd, WALKey logKey, + WALEdit logEdit) { + this.appends++; + if (this.appends % whenToRoll == 0) { + LOG.info("Rolling after " + appends + " edits"); + // We used to do explicit call to rollWriter but changed it to a request + // to avoid dead lock (there are less threads going on in this class than + // in the regionserver -- regionserver does not have the issue). + // TODO I think this means no rolling actually happens; the request relies on there + // being a LogRoller. + DefaultWALProvider.requestLogRoll(wal); + } } - } - @Override - public void postSync(final long timeInNanos, final int handlerSyncs) { - syncMeter.mark(); - syncHistogram.update(timeInNanos); - syncCountHistogram.update(handlerSyncs); - } + @Override + public void postSync(final long timeInNanos, final int handlerSyncs) { + syncMeter.mark(); + syncHistogram.update(timeInNanos); + syncCountHistogram.update(handlerSyncs); + } - @Override - public void postAppend(final long size, final long elapsedTime) { - appendMeter.mark(size); - } - }); - wal.rollWriter(); + @Override + public void postAppend(final long size, final long elapsedTime) { + appendMeter.mark(size); + } + }); + wal.rollWriter(); + } return HRegion.createHRegion(regionInfo, dir, getConf(), htd, wal); }