MAPREDUCE-7309. Improve performance of reading resource request for mapper/reducers from config. Contributed by Peter Bacsko & Wangda Tan.
This commit is contained in:
parent
9dd74141a6
commit
8ed565382f
|
@ -158,6 +158,9 @@ public abstract class TaskAttemptImpl implements
|
|||
org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt,
|
||||
EventHandler<TaskAttemptEvent> {
|
||||
|
||||
@VisibleForTesting
|
||||
protected final static Map<TaskType, Resource> RESOURCE_REQUEST_CACHE
|
||||
= new HashMap<>();
|
||||
static final Counters EMPTY_COUNTERS = new Counters();
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(TaskAttemptImpl.class);
|
||||
|
@ -172,7 +175,7 @@ public abstract class TaskAttemptImpl implements
|
|||
private final Clock clock;
|
||||
private final org.apache.hadoop.mapred.JobID oldJobId;
|
||||
private final TaskAttemptListener taskAttemptListener;
|
||||
private final Resource resourceCapability;
|
||||
private Resource resourceCapability;
|
||||
protected Set<String> dataLocalHosts;
|
||||
protected Set<String> dataLocalRacks;
|
||||
private final List<String> diagnostics = new ArrayList<String>();
|
||||
|
@ -707,6 +710,10 @@ public abstract class TaskAttemptImpl implements
|
|||
getResourceTypePrefix(taskType);
|
||||
boolean memorySet = false;
|
||||
boolean cpuVcoresSet = false;
|
||||
if (RESOURCE_REQUEST_CACHE.get(taskType) != null) {
|
||||
resourceCapability = RESOURCE_REQUEST_CACHE.get(taskType);
|
||||
return;
|
||||
}
|
||||
if (resourceTypePrefix != null) {
|
||||
List<ResourceInformation> resourceRequests =
|
||||
ResourceUtils.getRequestedResourcesFromConfig(conf,
|
||||
|
@ -767,6 +774,9 @@ public abstract class TaskAttemptImpl implements
|
|||
if (!cpuVcoresSet) {
|
||||
this.resourceCapability.setVirtualCores(getCpuRequired(conf, taskType));
|
||||
}
|
||||
RESOURCE_REQUEST_CACHE.put(taskType, resourceCapability);
|
||||
LOG.info("Resource capability of task type {} is set to {}",
|
||||
taskType, resourceCapability);
|
||||
}
|
||||
|
||||
private String getCpuVcoresKey(TaskType taskType) {
|
||||
|
|
|
@ -43,6 +43,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
|
|||
import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
|
@ -155,6 +156,11 @@ public class TestTaskAttempt{
|
|||
ResourceUtils.resetResourceTypes(new Configuration());
|
||||
}
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear();
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() {
|
||||
ResourceUtils.resetResourceTypes(new Configuration());
|
||||
|
@ -1721,6 +1727,7 @@ public class TestTaskAttempt{
|
|||
TestAppender testAppender = new TestAppender();
|
||||
final Logger logger = Logger.getLogger(TaskAttemptImpl.class);
|
||||
try {
|
||||
TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear();
|
||||
logger.addAppender(testAppender);
|
||||
EventHandler eventHandler = mock(EventHandler.class);
|
||||
Clock clock = SystemClock.getInstance();
|
||||
|
|
Loading…
Reference in New Issue