From 7e622076d41a85fc9a8600fb270564a085f5cd83 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Wed, 8 Apr 2015 15:39:25 -0500 Subject: [PATCH] HDFS-8046. Allow better control of getContentSummary. Contributed by Kihwal Lee. (cherry picked from commit 285b31e75e51ec8e3a796c2cb0208739368ca9b8) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +++- .../namenode/ContentSummaryComputationContext.java | 10 +++++++--- .../hdfs/server/namenode/FSDirStatAndListingOp.java | 2 +- .../hadoop/hdfs/server/namenode/FSDirectory.java | 8 ++++++++ 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7736f62cbce..e7af8dccdf7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -79,6 +79,8 @@ Release 2.8.0 - UNRELEASED HDFS-8085. Move CorruptFileBlockIterator to a new hdfs.client.impl package. (szetszwo) + HDFS-8046. Allow better control of getContentSummary (kihwal) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 50e4b33d569..a8dfb02421e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -200,7 +200,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_LIST_LIMIT = "dfs.ls.limit"; public static final int DFS_LIST_LIMIT_DEFAULT = 1000; public static final String DFS_CONTENT_SUMMARY_LIMIT_KEY = "dfs.content-summary.limit"; - public static final int DFS_CONTENT_SUMMARY_LIMIT_DEFAULT = 0; + public static final int DFS_CONTENT_SUMMARY_LIMIT_DEFAULT = 5000; + public static final String DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_KEY = "dfs.content-summary.sleep-microsec"; + public static final long DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT = 500; public static final String DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated"; public static final int DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT = 0; public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java index 31f34b9133d..5739835acf4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java @@ -32,6 +32,8 @@ public class ContentSummaryComputationContext { private long nextCountLimit = 0; private long limitPerRun = 0; private long yieldCount = 0; + private long sleepMilliSec = 0; + private int sleepNanoSec = 0; /** * Constructor @@ -43,17 +45,19 @@ public class ContentSummaryComputationContext { * no limit (i.e. no yielding) */ public ContentSummaryComputationContext(FSDirectory dir, - FSNamesystem fsn, long limitPerRun) { + FSNamesystem fsn, long limitPerRun, long sleepMicroSec) { this.dir = dir; this.fsn = fsn; this.limitPerRun = limitPerRun; this.nextCountLimit = limitPerRun; this.counts = new ContentCounts.Builder().build(); + this.sleepMilliSec = sleepMicroSec/1000; + this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000); } /** Constructor for blocking computation. */ public ContentSummaryComputationContext(BlockStoragePolicySuite bsps) { - this(null, null, 0); + this(null, null, 0, 1000); this.bsps = bsps; } @@ -105,7 +109,7 @@ public class ContentSummaryComputationContext { fsn.readUnlock(); try { - Thread.sleep(1); + Thread.sleep(sleepMilliSec, sleepNanoSec); } catch (InterruptedException ie) { } finally { // reacquire diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index 43c2de3c471..850b3bd6c51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -496,7 +496,7 @@ class FSDirStatAndListingOp { // processed. 0 means disabled. I.e. blocking for the entire duration. ContentSummaryComputationContext cscc = new ContentSummaryComputationContext(fsd, fsd.getFSNamesystem(), - fsd.getContentCountLimit()); + fsd.getContentCountLimit(), fsd.getContentSleepMicroSec()); ContentSummary cs = targetNode.computeAndConvertContentSummary(cscc); fsd.addYieldCount(cscc.getYieldCount()); return cs; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 7eea343fdcb..966cf3ae5b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -136,6 +136,7 @@ public class FSDirectory implements Closeable { private final int maxDirItems; private final int lsLimit; // max list limit private final int contentCountLimit; // max content summary counts per run + private final long contentSleepMicroSec; private final INodeMap inodeMap; // Synchronized by dirLock private long yieldCount = 0; // keep track of lock yield count. @@ -264,6 +265,9 @@ public class FSDirectory implements Closeable { this.contentCountLimit = conf.getInt( DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY, DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_DEFAULT); + this.contentSleepMicroSec = conf.getLong( + DFSConfigKeys.DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_KEY, + DFSConfigKeys.DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT); // filesystem limits this.maxComponentLength = conf.getInt( @@ -345,6 +349,10 @@ public class FSDirectory implements Closeable { return contentCountLimit; } + long getContentSleepMicroSec() { + return contentSleepMicroSec; + } + int getInodeXAttrsLimit() { return inodeXAttrsLimit; }