From d0b2c5850b523a3888b2fadcfcdf6edbed33f221 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Sun, 24 Sep 2017 21:03:06 -0700 Subject: [PATCH] HDFS-12516. Suppress the fsnamesystem lock warning on nn startup. Contributed by Ajay Kumar. --- .../hdfs/server/namenode/FSNamesystem.java | 7 ++- .../server/namenode/FSNamesystemLock.java | 27 +++++++++-- .../server/namenode/TestFSNamesystemLock.java | 47 ++++++++++++++++++- 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 967f26b4482..217df941a52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1095,7 +1095,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, if (!success) { fsImage.close(); } - writeUnlock("loadFSImage"); + writeUnlock("loadFSImage", true); } imageLoadComplete(); } @@ -1586,6 +1586,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, public void writeUnlock(String opName) { this.fsLock.writeUnlock(opName); } + + public void writeUnlock(String opName, boolean suppressWriteLockReport) { + this.fsLock.writeUnlock(opName, suppressWriteLockReport); + } + @Override public boolean hasWriteLock() { return this.fsLock.isWriteLockedByCurrentThread(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java index 8c60faac3fe..32c7efa5c2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java @@ -207,13 +207,34 @@ class FSNamesystemLock { } } + /** + * Unlocks FSNameSystem write lock. This internally calls {@link + * FSNamesystemLock#writeUnlock(String, boolean)} + */ public void writeUnlock() { - writeUnlock(OP_NAME_OTHER); + writeUnlock(OP_NAME_OTHER, false); } + /** + * Unlocks FSNameSystem write lock. This internally calls {@link + * FSNamesystemLock#writeUnlock(String, boolean)} + * + * @param opName Operation name. + */ public void writeUnlock(String opName) { - final boolean needReport = coarseLock.getWriteHoldCount() == 1 && - coarseLock.isWriteLockedByCurrentThread(); + writeUnlock(opName, false); + } + + /** + * Unlocks FSNameSystem write lock. + * + * @param opName Operation name + * @param suppressWriteLockReport When false, event of write lock being held + * for long time will be logged in logs and metrics. + */ + public void writeUnlock(String opName, boolean suppressWriteLockReport) { + final boolean needReport = !suppressWriteLockReport && coarseLock + .getWriteHoldCount() == 1 && coarseLock.isWriteLockedByCurrentThread(); final long currentTimeNanos = timer.monotonicNowNanos(); final long writeLockIntervalNanos = currentTimeNanos - writeLockHeldTimeStampNanos; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java index 7f9746d8f82..94e0b333a0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java @@ -38,6 +38,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.regex.Pattern; +import org.slf4j.LoggerFactory; import static org.junit.Assert.*; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY; @@ -347,7 +348,7 @@ public class TestFSNamesystemLock { fsLock.writeLock(); timer.advance(1); - fsLock.writeUnlock("baz"); + fsLock.writeUnlock("baz", false); MetricsRecordBuilder rb = MetricsAsserts.mockMetricsRecordBuilder(); rates.snapshot(rb, true); @@ -360,4 +361,48 @@ public class TestFSNamesystemLock { assertCounter("FSNWriteLockBazNanosNumOps", 1L, rb); } + /** + * Test to suppress FSNameSystem write lock report when it is held for long + * time. + */ + @Test(timeout = 45000) + public void testFSWriteLockReportSuppressed() throws Exception { + final long writeLockReportingThreshold = 1L; + final long writeLockSuppressWarningInterval = 10L; + Configuration conf = new Configuration(); + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY, + writeLockReportingThreshold); + conf.setTimeDuration(DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, + writeLockSuppressWarningInterval, TimeUnit.MILLISECONDS); + + final FakeTimer timer = new FakeTimer(); + final FSNamesystemLock fsnLock = new FSNamesystemLock(conf, null, timer); + timer.advance(writeLockSuppressWarningInterval); + + LogCapturer logs = LogCapturer.captureLogs(FSNamesystem.LOG); + GenericTestUtils + .setLogLevel(LoggerFactory.getLogger(FSNamesystem.class.getName()), + org.slf4j.event.Level.INFO); + + // Should trigger the write lock report + fsnLock.writeLock(); + timer.advance(writeLockReportingThreshold + 100); + fsnLock.writeUnlock(); + assertTrue(logs.getOutput().contains( + "FSNamesystem write lock held for")); + + logs.clearOutput(); + + // Suppress report if the write lock is held for a long time + fsnLock.writeLock(); + timer.advance(writeLockReportingThreshold + 100); + fsnLock.writeUnlock("testFSWriteLockReportSuppressed", true); + assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); + assertFalse(logs.getOutput().contains( + "Number of suppressed write-lock reports:")); + assertFalse(logs.getOutput().contains( + "FSNamesystem write lock held for")); + } + } \ No newline at end of file