From f3a4cbcc60572c73e0d15580bae42a36c92fe247 Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Fri, 26 Aug 2016 14:19:55 -0700 Subject: [PATCH] HDFS-10798. Make the threshold of reporting FSNamesystem lock contention configurable. Contributed by Erik Krogen. (cherry picked from commit 407b519fb14f79f19ebc4fbdf08204336a7acf77) --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +++++ .../hadoop/hdfs/server/namenode/FSNamesystem.java | 10 ++++++++-- .../src/main/resources/hdfs-default.xml | 9 +++++++++ .../hdfs/server/namenode/TestFSNamesystem.java | 15 +++++++++------ 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index d7b19afff8b..3da3807b9e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -384,6 +384,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT = 25; + // Threshold for how long a write lock must be held for the event to be logged + public static final String DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY = + "dfs.namenode.write-lock-reporting-threshold-ms"; + public static final long DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 1000L; + public static final String DFS_UPGRADE_DOMAIN_FACTOR = "dfs.namenode.upgrade.domain.factor"; public static final int DFS_UPGRADE_DOMAIN_FACTOR_DEFAULT = DFS_REPLICATION_DEFAULT; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ef4b6c06b43..51c41a425ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -71,6 +71,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_DEFAULT; @@ -807,6 +809,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY, DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT); + this.writeLockReportingThreshold = conf.getLong( + DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY, + DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT); + // For testing purposes, allow the DT secret manager to be started regardless // of whether security is enabled. alwaysUseDelegationTokensForTests = conf.getBoolean( @@ -1486,7 +1492,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } /** Threshold (ms) for long holding write lock report. */ - static final short WRITELOCK_REPORTING_THRESHOLD = 1000; + private long writeLockReportingThreshold; /** Last time stamp for write lock. Keep the longest one for multi-entrance.*/ private long writeLockHeldTimeStamp; @@ -1520,7 +1526,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, this.fsLock.writeLock().unlock(); - if (needReport && writeLockInterval >= WRITELOCK_REPORTING_THRESHOLD) { + if (needReport && writeLockInterval >= this.writeLockReportingThreshold) { LOG.info("FSNamesystem write lock held for " + writeLockInterval + " ms via\n" + StringUtils.getStackTrace(Thread.currentThread())); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 38bfedd3ac3..c9f22879d95 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2626,6 +2626,15 @@ + + dfs.namenode.write-lock-reporting-threshold-ms + 1000 + When a write lock is held on the namenode for a long time, + this will be logged as the lock is released. This sets how long the + lock must be held for logging to occur. + + + dfs.namenode.startup.delay.block.deletion.sec 0 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java index 572b40d0f3d..df9001da27d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -290,7 +290,10 @@ public class TestFSNamesystem { */ @Test(timeout=45000) public void testFSLockLongHoldingReport() throws Exception { + final long writeLockReportingThreshold = 100L; Configuration conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY, + writeLockReportingThreshold); FSImage fsImage = Mockito.mock(FSImage.class); FSEditLog fsEditLog = Mockito.mock(FSEditLog.class); Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog); @@ -301,32 +304,32 @@ public class TestFSNamesystem { // Don't report if the write lock is held for a short time fsn.writeLock(); - Thread.sleep(FSNamesystem.WRITELOCK_REPORTING_THRESHOLD / 2); + Thread.sleep(writeLockReportingThreshold / 2); fsn.writeUnlock(); assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName())); // Report if the write lock is held for a long time fsn.writeLock(); - Thread.sleep(FSNamesystem.WRITELOCK_REPORTING_THRESHOLD + 100); + Thread.sleep(writeLockReportingThreshold + 10); logs.clearOutput(); fsn.writeUnlock(); assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName())); // Report if the write lock is held (interruptibly) for a long time fsn.writeLockInterruptibly(); - Thread.sleep(FSNamesystem.WRITELOCK_REPORTING_THRESHOLD + 100); + Thread.sleep(writeLockReportingThreshold + 10); logs.clearOutput(); fsn.writeUnlock(); assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName())); // Report if it's held for a long time when re-entering write lock fsn.writeLock(); - Thread.sleep(FSNamesystem.WRITELOCK_REPORTING_THRESHOLD / 2 + 1); + Thread.sleep(writeLockReportingThreshold/ 2 + 1); fsn.writeLockInterruptibly(); - Thread.sleep(FSNamesystem.WRITELOCK_REPORTING_THRESHOLD / 2 + 1); + Thread.sleep(writeLockReportingThreshold / 2 + 1); fsn.writeLock(); - Thread.sleep(FSNamesystem.WRITELOCK_REPORTING_THRESHOLD / 2); + Thread.sleep(writeLockReportingThreshold / 2); logs.clearOutput(); fsn.writeUnlock(); assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()));