MAPREDUCE-5601. ShuffleHandler fadvises file regions as DONTNEED even when fetch fails (Sandy Ryza)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1537855 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanford Ryza 2013-11-01 08:51:54 +00:00
parent da317f2ea5
commit 86dc50c1eb
3 changed files with 14 additions and 1 deletions

View File

@ -171,6 +171,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again
in Limits (Sandy Ryza)
MAPREDUCE-5601. ShuffleHandler fadvises file regions as DONTNEED even when
fetch fails (Sandy Ryza)
BUG FIXES
MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal

View File

@ -69,6 +69,14 @@ public class FadvisedFileRegion extends DefaultFileRegion {
if (readaheadRequest != null) {
readaheadRequest.cancel();
}
super.releaseExternalResources();
}
/**
* Call when the transfer completes successfully so we can advise the OS that
* we don't need the region to be cached anymore.
*/
public void transferSuccessful() {
if (manageOsCache && getCount() > 0) {
try {
NativeIO.POSIX.posixFadviseIfPossible(identifier,
@ -78,6 +86,5 @@ public class FadvisedFileRegion extends DefaultFileRegion {
LOG.warn("Failed to manage OS cache for " + identifier, t);
}
}
super.releaseExternalResources();
}
}

View File

@ -625,6 +625,9 @@ public class ShuffleHandler extends AuxiliaryService {
// attribute to appropriate spill output
@Override
public void operationComplete(ChannelFuture future) {
if (future.isSuccess()) {
partition.transferSuccessful();
}
partition.releaseExternalResources();
}
});