Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

This commit is contained in:
Yufei Gu 2017-10-20 09:59:07 -07:00
parent b61144a93d
commit 0799fde35e
2 changed files with 25 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
View File

@ -350,6 +350,11 @@ public class FSDownload implements Callable<Path> {
} catch (URISyntaxException e) { } catch (URISyntaxException e) {
throw new IOException("Invalid resource", e); throw new IOException("Invalid resource", e);
} }
if (LOG.isDebugEnabled()) {
LOG.debug("Starting to download " + sCopy);
}
createDir(destDirPath, cachePerms); createDir(destDirPath, cachePerms);
final Path dst_work = new Path(destDirPath + "_tmp"); final Path dst_work = new Path(destDirPath + "_tmp");
createDir(dst_work, cachePerms); createDir(dst_work, cachePerms);
@ -364,6 +369,11 @@ public class FSDownload implements Callable<Path> {
unpack(new File(dTmp.toUri()), new File(dFinal.toUri())); unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
changePermissions(dFinal.getFileSystem(conf), dFinal); changePermissions(dFinal.getFileSystem(conf), dFinal);
files.rename(dst_work, destDirPath, Rename.OVERWRITE); files.rename(dst_work, destDirPath, Rename.OVERWRITE);
if (LOG.isDebugEnabled()) {
LOG.debug("File has been downloaded to " +
new Path(destDirPath, sCopy.getName()));
}
} catch (Exception e) { } catch (Exception e) {
try { try {
files.delete(destDirPath, true); files.delete(destDirPath, true);
@ -409,8 +419,11 @@ public class FSDownload implements Callable<Path> {
// APPLICATION: // APPLICATION:
perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS; perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
} }
LOG.debug("Changing permissions for path " + path
+ " to perm " + perm); if (LOG.isDebugEnabled()) {
LOG.debug("Changing permissions for path " + path + " to perm " + perm);
}
final FsPermission fPerm = perm; final FsPermission fPerm = perm;
if (null == userUgi) { if (null == userUgi) {
files.setPermission(path, perm); files.setPermission(path, perm);

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
View File

@ -859,7 +859,7 @@ public class ResourceLocalizationService extends CompositeService
// TODO handle failures, cancellation, requests by other containers // TODO handle failures, cancellation, requests by other containers
LocalizedResource rsrc = request.getResource(); LocalizedResource rsrc = request.getResource();
LocalResourceRequest key = rsrc.getRequest(); LocalResourceRequest key = rsrc.getRequest();
LOG.info("Downloading public rsrc:" + key); LOG.info("Downloading public resource: " + key);
/* /*
* Here multiple containers may request the same resource. So we need * Here multiple containers may request the same resource. So we need
* to start downloading only when * to start downloading only when
@ -918,8 +918,17 @@ public class ResourceLocalizationService extends CompositeService
+ " Either queue is full or threadpool is shutdown.", re); + " Either queue is full or threadpool is shutdown.", re);
} }
} else { } else {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip downloading resource: " + key + " since it's in"
+ " state: " + rsrc.getState());
}
rsrc.unlock(); rsrc.unlock();
} }
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip downloading resource: " + key + " since it is locked"
+ " by other threads");
}
} }
} }