Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery. Contributed by Todd Lipcon. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1562969 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2014-01-30 21:21:02 +00:00
parent bf1487b457
commit dff75d1e4b
3 changed files with 7 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -13,6 +13,9 @@ Release 2.4.0 - UNRELEASED
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery
(todd)
BUG FIXES
HDFS-5492. Port HDFS-2069 (Incorrect default trash interval in the

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -3773,7 +3773,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
if (diff > 0) {
try {
String path = leaseManager.findPath(fileINode);
String path = fileINode.getFullPathName();
dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication());
} catch (IOException e) {
LOG.warn("Unexpected exception while updating disk space.", e);
@ -3978,7 +3978,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
BlockInfo storedBlock)
throws IOException {
String src = leaseManager.findPath(pendingFile);
String src = pendingFile.getFullPathName();
// commit the last block and complete it if it has minimum replicas
commitOrCompleteLastBlock(pendingFile, storedBlock);
@ -4000,7 +4000,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
@VisibleForTesting
String persistBlocks(INodeFileUnderConstruction pendingFile,
boolean logRetryCache) throws IOException {
String src = leaseManager.findPath(pendingFile);
String src = pendingFile.getFullPathName();
dir.persistBlocks(src, pendingFile, logRetryCache);
return src;
}
@ -5929,7 +5929,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
.getDatanodeStorageInfos(newNodes, newStorageIDs);
blockinfo.setExpectedLocations(storages);
String src = leaseManager.findPath(pendingFile);
String src = pendingFile.getFullPathName();
dir.persistBlocks(src, pendingFile, logRetryCache);
}

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
View File

@ -179,22 +179,6 @@ public class LeaseManager {
return addLease(newHolder, src);
}
/**
* Finds the pathname for the specified pendingFile
*/
public synchronized String findPath(INodeFileUnderConstruction pendingFile)
throws IOException {
Lease lease = getLease(pendingFile.getClientName());
if (lease != null) {
String src = lease.findPath(pendingFile);
if (src != null) {
return src;
}
}
throw new IOException("pendingFile (=" + pendingFile + ") not found."
+ "(lease=" + lease + ")");
}
/**
* Renew the lease(s) held by the given client
*/
@ -250,24 +234,6 @@ public class LeaseManager {
return now() - lastUpdate > softLimit;
}
/**
* @return the path associated with the pendingFile and null if not found.
*/
private String findPath(INodeFileUnderConstruction pendingFile) {
try {
for (String src : paths) {
INode node = fsnamesystem.dir.getINode(src);
if (node == pendingFile
|| (node.isFile() && node.asFile() == pendingFile)) {
return src;
}
}
} catch (UnresolvedLinkException e) {
throw new AssertionError("Lease files should reside on this FS");
}
return null;
}
/** Does this lease contain any path? */
boolean hasPath() {return !paths.isEmpty();}