Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11345. Document the configuration key for FSNamesystem lock fairness. Contributed by Erik Krogen. 

(cherry picked from commit 2c769167db)
(cherry picked from commit 6b3c13d543)
This commit is contained in:
Akira Ajisaka 2017-06-20 16:35:54 +09:00
parent 978ba21bd0
commit aa5d4330b2
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
5 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -139,6 +139,9 @@ Release 2.7.4 - UNRELEASED
HDFS-8674. Improve performance of postponed block scans. (Daryn Sharp)
HDFS-11345. Document the configuration key for FSNamesystem lock
fairness. (Erik Krogen via aajisaka)
BUG FIXES
HDFS-8307. Spurious DNS Queries from hdfs shell. (Andres Perez via aengineer)

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -377,6 +377,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_MAX_XATTR_SIZE_KEY = "dfs.namenode.fs-limits.max-xattr-size";
public static final int DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT = 16384;
public static final String DFS_NAMENODE_FSLOCK_FAIR_KEY =
"dfs.namenode.fslock.fair";
public static final boolean DFS_NAMENODE_FSLOCK_FAIR_DEFAULT = true;
public static final String DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY =
"dfs.namenode.lock.detailed-metrics.enabled";
public static final boolean DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT =

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
View File

@ -32,6 +32,8 @@ import org.apache.hadoop.util.Timer;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
@ -113,7 +115,8 @@ class FSNamesystemLock {
@VisibleForTesting
FSNamesystemLock(Configuration conf,
MutableRatesWithAggregation detailedHoldTimeMetrics, Timer timer) {
boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
boolean fair = conf.getBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY,
DFS_NAMENODE_FSLOCK_FAIR_DEFAULT);
FSNamesystem.LOG.info("fsLock is fair: " + fair);
this.coarseLock = new ReentrantReadWriteLock(fair);
this.timer = timer;

10
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -2391,6 +2391,16 @@
</description>
</property>
<property>
<name>dfs.namenode.fslock.fair</name>
<value>true</value>
<description>If this is true, the FS Namesystem lock will be used in Fair mode,
which will help to prevent writer threads from being starved, but can provide
lower lock throughput. See java.util.concurrent.locks.ReentrantReadWriteLock
for more information on fair/non-fair locks.
</description>
</property>
<property>
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
<value>0</value>

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
View File

@ -40,6 +40,7 @@ import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
import static org.junit.Assert.*;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
@ -53,11 +54,11 @@ public class TestFSNamesystemLock {
public void testFsLockFairness() throws IOException, InterruptedException{
Configuration conf = new Configuration();
conf.setBoolean("dfs.namenode.fslock.fair", true);
conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, true);
FSNamesystemLock fsnLock = new FSNamesystemLock(conf, null);
assertTrue(fsnLock.coarseLock.isFair());
conf.setBoolean("dfs.namenode.fslock.fair", false);
conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, false);
fsnLock = new FSNamesystemLock(conf, null);
assertFalse(fsnLock.coarseLock.isFair());
}
@ -103,7 +104,7 @@ public class TestFSNamesystemLock {
final int threadCount = 3;
final CountDownLatch latch = new CountDownLatch(threadCount);
final Configuration conf = new Configuration();
conf.setBoolean("dfs.namenode.fslock.fair", true);
conf.setBoolean(DFS_NAMENODE_FSLOCK_FAIR_KEY, true);
final FSNamesystemLock rwLock = new FSNamesystemLock(conf, null);
rwLock.writeLock();
ExecutorService helper = Executors.newFixedThreadPool(threadCount);