MAPREDUCE-5270. Migrated MR app from using BuilderUtil factory methods to individual record factory methods. Contributed by Jian He.
svn merge --ignore-ancestry -c 1486271 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1487158 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
ea60ea3b85
commit
a2303831b4
|
@ -96,6 +96,9 @@ Release 2.0.5-beta - UNRELEASED
|
||||||
MAPREDUCE-5230. Bring back NLineInputFormat.createFileSplit for binary
|
MAPREDUCE-5230. Bring back NLineInputFormat.createFileSplit for binary
|
||||||
compatibility with mapred in 1.x (Mayank Bansal via vinodkv)
|
compatibility with mapred in 1.x (Mayank Bansal via vinodkv)
|
||||||
|
|
||||||
|
MAPREDUCE-5270. Migrated MR app from using BuilderUtil factory methods to
|
||||||
|
individual record factory methods. (Jian He via vinodkv)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
|
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
|
||||||
|
|
|
@ -132,7 +132,6 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
|
||||||
import org.apache.hadoop.yarn.state.StateMachine;
|
import org.apache.hadoop.yarn.state.StateMachine;
|
||||||
import org.apache.hadoop.yarn.state.StateMachineFactory;
|
import org.apache.hadoop.yarn.state.StateMachineFactory;
|
||||||
import org.apache.hadoop.yarn.util.Apps;
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.RackResolver;
|
import org.apache.hadoop.yarn.util.RackResolver;
|
||||||
|
|
||||||
|
@ -598,8 +597,8 @@ public abstract class TaskAttemptImpl implements
|
||||||
long resourceSize = fstat.getLen();
|
long resourceSize = fstat.getLen();
|
||||||
long resourceModificationTime = fstat.getModificationTime();
|
long resourceModificationTime = fstat.getModificationTime();
|
||||||
|
|
||||||
return BuilderUtils.newLocalResource(resourceURL, type, visibility,
|
return LocalResource.newInstance(resourceURL, type, visibility,
|
||||||
resourceSize, resourceModificationTime);
|
resourceSize, resourceModificationTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -762,10 +761,9 @@ public abstract class TaskAttemptImpl implements
|
||||||
// Construct the actual Container
|
// Construct the actual Container
|
||||||
// The null fields are per-container and will be constructed for each
|
// The null fields are per-container and will be constructed for each
|
||||||
// container separately.
|
// container separately.
|
||||||
ContainerLaunchContext container = BuilderUtils
|
ContainerLaunchContext container =
|
||||||
.newContainerLaunchContext(localResources,
|
ContainerLaunchContext.newInstance(localResources, environment, null,
|
||||||
environment, null, serviceData, taskCredentialsBuffer,
|
serviceData, taskCredentialsBuffer, applicationACLs);
|
||||||
applicationACLs);
|
|
||||||
|
|
||||||
return container;
|
return container;
|
||||||
}
|
}
|
||||||
|
@ -806,7 +804,7 @@ public abstract class TaskAttemptImpl implements
|
||||||
}
|
}
|
||||||
|
|
||||||
// Construct the actual Container
|
// Construct the actual Container
|
||||||
ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext(
|
ContainerLaunchContext container = ContainerLaunchContext.newInstance(
|
||||||
commonContainerSpec.getLocalResources(), myEnv, commands,
|
commonContainerSpec.getLocalResources(), myEnv, commands,
|
||||||
myServiceData, commonContainerSpec.getTokens().duplicate(),
|
myServiceData, commonContainerSpec.getTokens().duplicate(),
|
||||||
applicationACLs);
|
applicationACLs);
|
||||||
|
@ -1096,7 +1094,7 @@ public abstract class TaskAttemptImpl implements
|
||||||
// launching the container on an NM, these are already completed tasks, so
|
// launching the container on an NM, these are already completed tasks, so
|
||||||
// setting them to null and RMIdentifier as 0
|
// setting them to null and RMIdentifier as 0
|
||||||
container =
|
container =
|
||||||
BuilderUtils.newContainer(containerId, containerNodeId,
|
Container.newInstance(containerId, containerNodeId,
|
||||||
nodeHttpAddress, null, null, null, 0);
|
nodeHttpAddress, null, null, null, 0);
|
||||||
computeRackAndLocality();
|
computeRackAndLocality();
|
||||||
launchTime = taInfo.getStartTime();
|
launchTime = taInfo.getStartTime();
|
||||||
|
|
|
@ -239,7 +239,11 @@ public abstract class RMContainerRequestor extends RMCommunicator {
|
||||||
// if ask already sent to RM, we can try and overwrite it if possible.
|
// if ask already sent to RM, we can try and overwrite it if possible.
|
||||||
// send a new ask to RM with numContainers
|
// send a new ask to RM with numContainers
|
||||||
// specified for the blacklisted host to be 0.
|
// specified for the blacklisted host to be 0.
|
||||||
ResourceRequest zeroedRequest = BuilderUtils.newResourceRequest(req);
|
ResourceRequest zeroedRequest =
|
||||||
|
ResourceRequest.newInstance(req.getPriority(),
|
||||||
|
req.getHostName(), req.getCapability(),
|
||||||
|
req.getNumContainers());
|
||||||
|
|
||||||
zeroedRequest.setNumContainers(0);
|
zeroedRequest.setNumContainers(0);
|
||||||
// to be sent to RM on next heartbeat
|
// to be sent to RM on next heartbeat
|
||||||
addResourceRequestToAsk(zeroedRequest);
|
addResourceRequestToAsk(zeroedRequest);
|
||||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.hadoop.http.HttpConfig;
|
||||||
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
|
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
|
|
||||||
@XmlRootElement(name = "jobAttempt")
|
@XmlRootElement(name = "jobAttempt")
|
||||||
@XmlAccessorType(XmlAccessType.FIELD)
|
@XmlAccessorType(XmlAccessType.FIELD)
|
||||||
|
@ -53,7 +52,7 @@ public class AMAttemptInfo {
|
||||||
int nmPort = amInfo.getNodeManagerPort();
|
int nmPort = amInfo.getNodeManagerPort();
|
||||||
if (nmHost != null) {
|
if (nmHost != null) {
|
||||||
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
|
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
|
||||||
NodeId nodeId = BuilderUtils.newNodeId(nmHost, nmPort);
|
NodeId nodeId = NodeId.newInstance(nmHost, nmPort);
|
||||||
this.nodeId = nodeId.toString();
|
this.nodeId = nodeId.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -272,8 +272,8 @@ public class TestJobHistoryEventHandler {
|
||||||
String workDir = setupTestWorkDir();
|
String workDir = setupTestWorkDir();
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
|
ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
BuilderUtils.newApplicationAttemptId(appId, 1);
|
ApplicationAttemptId.newInstance(appId, 1);
|
||||||
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
|
ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
|
||||||
TaskID taskID = TaskID.forName("task_200707121733_0003_m_000005");
|
TaskID taskID = TaskID.forName("task_200707121733_0003_m_000005");
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
||||||
AppContext mockAppContext = mockAppContext(appId);
|
AppContext mockAppContext = mockAppContext(appId);
|
||||||
|
|
|
@ -91,13 +91,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.Container;
|
import org.apache.hadoop.yarn.api.records.Container;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.service.Service;
|
import org.apache.hadoop.yarn.service.Service;
|
||||||
import org.apache.hadoop.yarn.state.StateMachine;
|
import org.apache.hadoop.yarn.state.StateMachine;
|
||||||
import org.apache.hadoop.yarn.state.StateMachineFactory;
|
import org.apache.hadoop.yarn.state.StateMachineFactory;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -164,7 +164,7 @@ public class MRApp extends MRAppMaster {
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
getApplicationAttemptId(applicationId, startCount);
|
getApplicationAttemptId(applicationId, startCount);
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
BuilderUtils.newContainerId(appAttemptId, startCount);
|
ContainerId.newInstance(appAttemptId, startCount);
|
||||||
return containerId;
|
return containerId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -231,9 +231,9 @@ public class MRApp extends MRAppMaster {
|
||||||
this.clusterInfo.getMaxContainerCapability());
|
this.clusterInfo.getMaxContainerCapability());
|
||||||
} else {
|
} else {
|
||||||
getContext().getClusterInfo().setMinContainerCapability(
|
getContext().getClusterInfo().setMinContainerCapability(
|
||||||
BuilderUtils.newResource(1024, 1));
|
Resource.newInstance(1024, 1));
|
||||||
getContext().getClusterInfo().setMaxContainerCapability(
|
getContext().getClusterInfo().setMaxContainerCapability(
|
||||||
BuilderUtils.newResource(10240, 1));
|
Resource.newInstance(10240, 1));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -517,8 +517,8 @@ public class MRApp extends MRAppMaster {
|
||||||
ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
|
ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
|
||||||
cId.setApplicationAttemptId(getContext().getApplicationAttemptId());
|
cId.setApplicationAttemptId(getContext().getApplicationAttemptId());
|
||||||
cId.setId(containerCount++);
|
cId.setId(containerCount++);
|
||||||
NodeId nodeId = BuilderUtils.newNodeId(NM_HOST, NM_PORT);
|
NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT);
|
||||||
Container container = BuilderUtils.newContainer(cId, nodeId,
|
Container container = Container.newInstance(cId, nodeId,
|
||||||
NM_HOST + ":" + NM_HTTP_PORT, null, null, null, 0);
|
NM_HOST + ":" + NM_HTTP_PORT, null, null, null, 0);
|
||||||
JobID id = TypeConverter.fromYarn(applicationId);
|
JobID id = TypeConverter.fromYarn(applicationId);
|
||||||
JobId jobId = TypeConverter.toYarn(id);
|
JobId jobId = TypeConverter.toYarn(id);
|
||||||
|
|
|
@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
|
||||||
import org.apache.hadoop.yarn.api.records.Container;
|
import org.apache.hadoop.yarn.api.records.Container;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.api.records.ResourceRequest;
|
import org.apache.hadoop.yarn.api.records.ResourceRequest;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
|
@ -205,10 +206,10 @@ public class MRAppBenchmark {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
RegisterApplicationMasterResponse response =
|
RegisterApplicationMasterResponse response =
|
||||||
Records.newRecord(RegisterApplicationMasterResponse.class);
|
Records.newRecord(RegisterApplicationMasterResponse.class);
|
||||||
response.setMinimumResourceCapability(BuilderUtils
|
response.setMinimumResourceCapability(Resource.newInstance(
|
||||||
.newResource(1024, 1));
|
1024, 1));
|
||||||
response.setMaximumResourceCapability(BuilderUtils
|
response.setMaximumResourceCapability(Resource.newInstance(
|
||||||
.newResource(10240, 1));
|
10240, 1));
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -236,14 +237,13 @@ public class MRAppBenchmark {
|
||||||
int numContainers = req.getNumContainers();
|
int numContainers = req.getNumContainers();
|
||||||
for (int i = 0; i < numContainers; i++) {
|
for (int i = 0; i < numContainers; i++) {
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
BuilderUtils.newContainerId(
|
ContainerId.newInstance(
|
||||||
request.getApplicationAttemptId(),
|
request.getApplicationAttemptId(),
|
||||||
request.getResponseId() + i);
|
request.getResponseId() + i);
|
||||||
containers.add(BuilderUtils
|
containers.add(Container.newInstance(containerId,
|
||||||
.newContainer(containerId, BuilderUtils.newNodeId("host"
|
NodeId.newInstance("host" + containerId.getId(), 2345),
|
||||||
+ containerId.getId(), 2345),
|
"host" + containerId.getId() + ":5678",
|
||||||
"host" + containerId.getId() + ":5678", req
|
req.getCapability(), req.getPriority(), null, 0));
|
||||||
.getCapability(), req.getPriority(), null, 0));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -622,9 +622,9 @@ public class MockJobs extends MockApps {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static AMInfo createAMInfo(int attempt) {
|
private static AMInfo createAMInfo(int attempt) {
|
||||||
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
|
||||||
BuilderUtils.newApplicationId(100, 1), attempt);
|
BuilderUtils.newApplicationId(100, 1), attempt);
|
||||||
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
|
ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
|
||||||
return MRBuilderUtils.newAMInfo(appAttemptId, System.currentTimeMillis(),
|
return MRBuilderUtils.newAMInfo(appAttemptId, System.currentTimeMillis(),
|
||||||
containerId, NM_HOST, NM_PORT, NM_HTTP_PORT);
|
containerId, NM_HOST, NM_PORT, NM_HTTP_PORT);
|
||||||
}
|
}
|
||||||
|
|
|
@ -361,9 +361,9 @@ public class TestMRAppMaster {
|
||||||
|
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(12345, 56);
|
ApplicationId appId = BuilderUtils.newApplicationId(12345, 56);
|
||||||
ApplicationAttemptId applicationAttemptId =
|
ApplicationAttemptId applicationAttemptId =
|
||||||
BuilderUtils.newApplicationAttemptId(appId, 1);
|
ApplicationAttemptId.newInstance(appId, 1);
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
BuilderUtils.newContainerId(applicationAttemptId, 546);
|
ContainerId.newInstance(applicationAttemptId, 546);
|
||||||
String userName = UserGroupInformation.getCurrentUser().getShortUserName();
|
String userName = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||||
|
|
||||||
// Create staging dir, so MRAppMaster doesn't barf.
|
// Create staging dir, so MRAppMaster doesn't barf.
|
||||||
|
|
|
@ -496,7 +496,7 @@ public class TestRMContainerAllocator {
|
||||||
rm.sendAMLaunched(appAttemptId);
|
rm.sendAMLaunched(appAttemptId);
|
||||||
rmDispatcher.await();
|
rmDispatcher.await();
|
||||||
|
|
||||||
MRApp mrApp = new MRApp(appAttemptId, BuilderUtils.newContainerId(
|
MRApp mrApp = new MRApp(appAttemptId, ContainerId.newInstance(
|
||||||
appAttemptId, 0), 10, 10, false, this.getClass().getName(), true, 1) {
|
appAttemptId, 0), 10, 10, false, this.getClass().getName(), true, 1) {
|
||||||
@Override
|
@Override
|
||||||
protected Dispatcher createDispatcher() {
|
protected Dispatcher createDispatcher() {
|
||||||
|
@ -612,7 +612,7 @@ public class TestRMContainerAllocator {
|
||||||
MRApp mrApp, Task task) throws Exception {
|
MRApp mrApp, Task task) throws Exception {
|
||||||
TaskAttempt attempt = task.getAttempts().values().iterator().next();
|
TaskAttempt attempt = task.getAttempts().values().iterator().next();
|
||||||
List<ContainerStatus> contStatus = new ArrayList<ContainerStatus>(1);
|
List<ContainerStatus> contStatus = new ArrayList<ContainerStatus>(1);
|
||||||
contStatus.add(BuilderUtils.newContainerStatus(attempt.getAssignedContainerID(),
|
contStatus.add(ContainerStatus.newInstance(attempt.getAssignedContainerID(),
|
||||||
ContainerState.COMPLETE, "", 0));
|
ContainerState.COMPLETE, "", 0));
|
||||||
Map<ApplicationId,List<ContainerStatus>> statusUpdate =
|
Map<ApplicationId,List<ContainerStatus>> statusUpdate =
|
||||||
new HashMap<ApplicationId,List<ContainerStatus>>(1);
|
new HashMap<ApplicationId,List<ContainerStatus>>(1);
|
||||||
|
@ -648,7 +648,7 @@ public class TestRMContainerAllocator {
|
||||||
rm.sendAMLaunched(appAttemptId);
|
rm.sendAMLaunched(appAttemptId);
|
||||||
rmDispatcher.await();
|
rmDispatcher.await();
|
||||||
|
|
||||||
MRApp mrApp = new MRApp(appAttemptId, BuilderUtils.newContainerId(
|
MRApp mrApp = new MRApp(appAttemptId, ContainerId.newInstance(
|
||||||
appAttemptId, 0), 10, 0, false, this.getClass().getName(), true, 1) {
|
appAttemptId, 0), 10, 0, false, this.getClass().getName(), true, 1) {
|
||||||
@Override
|
@Override
|
||||||
protected Dispatcher createDispatcher() {
|
protected Dispatcher createDispatcher() {
|
||||||
|
@ -1229,7 +1229,7 @@ public class TestRMContainerAllocator {
|
||||||
List<ContainerId> release) {
|
List<ContainerId> release) {
|
||||||
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
|
List<ResourceRequest> askCopy = new ArrayList<ResourceRequest>();
|
||||||
for (ResourceRequest req : ask) {
|
for (ResourceRequest req : ask) {
|
||||||
ResourceRequest reqCopy = BuilderUtils.newResourceRequest(req
|
ResourceRequest reqCopy = ResourceRequest.newInstance(req
|
||||||
.getPriority(), req.getHostName(), req.getCapability(), req
|
.getPriority(), req.getHostName(), req.getCapability(), req
|
||||||
.getNumContainers());
|
.getNumContainers());
|
||||||
askCopy.add(reqCopy);
|
askCopy.add(reqCopy);
|
||||||
|
@ -1255,7 +1255,7 @@ public class TestRMContainerAllocator {
|
||||||
}
|
}
|
||||||
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
|
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
|
||||||
taskAttemptId);
|
taskAttemptId);
|
||||||
Resource containerNeed = BuilderUtils.newResource(memory, 1);
|
Resource containerNeed = Resource.newInstance(memory, 1);
|
||||||
if (earlierFailedAttempt) {
|
if (earlierFailedAttempt) {
|
||||||
return ContainerRequestEvent
|
return ContainerRequestEvent
|
||||||
.createContainerRequestEventForFailedContainer(attemptId,
|
.createContainerRequestEventForFailedContainer(attemptId,
|
||||||
|
@ -1338,8 +1338,8 @@ public class TestRMContainerAllocator {
|
||||||
when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
|
when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
|
||||||
when(context.getJob(isA(JobId.class))).thenReturn(job);
|
when(context.getJob(isA(JobId.class))).thenReturn(job);
|
||||||
when(context.getClusterInfo()).thenReturn(
|
when(context.getClusterInfo()).thenReturn(
|
||||||
new ClusterInfo(BuilderUtils.newResource(1024, 1), BuilderUtils
|
new ClusterInfo(Resource.newInstance(1024, 1), Resource.newInstance(
|
||||||
.newResource(10240, 1)));
|
10240, 1)));
|
||||||
when(context.getEventHandler()).thenReturn(new EventHandler() {
|
when(context.getEventHandler()).thenReturn(new EventHandler() {
|
||||||
@Override
|
@Override
|
||||||
public void handle(Event event) {
|
public void handle(Event event) {
|
||||||
|
@ -1412,12 +1412,12 @@ public class TestRMContainerAllocator {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Resource getMinContainerCapability() {
|
protected Resource getMinContainerCapability() {
|
||||||
return BuilderUtils.newResource(1024, 1);
|
return Resource.newInstance(1024, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Resource getMaxContainerCapability() {
|
protected Resource getMaxContainerCapability() {
|
||||||
return BuilderUtils.newResource(10240, 1);
|
return Resource.newInstance(10240, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void sendRequest(ContainerRequestEvent req) {
|
public void sendRequest(ContainerRequestEvent req) {
|
||||||
|
@ -1665,11 +1665,14 @@ public class TestRMContainerAllocator {
|
||||||
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(
|
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(
|
||||||
MRBuilderUtils.newTaskId(
|
MRBuilderUtils.newTaskId(
|
||||||
MRBuilderUtils.newJobId(1, 1, 1), 1, TaskType.MAP), 1);
|
MRBuilderUtils.newJobId(1, 1, 1), 1, TaskType.MAP), 1);
|
||||||
ContainerId containerId = BuilderUtils.newContainerId(1, 1, 1, 1);
|
ApplicationId applicationId = BuilderUtils.newApplicationId(1, 1);
|
||||||
ContainerStatus status = BuilderUtils.newContainerStatus(
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(
|
||||||
|
applicationId, 1);
|
||||||
|
ContainerId containerId = ContainerId.newInstance(applicationAttemptId, 1);
|
||||||
|
ContainerStatus status = ContainerStatus.newInstance(
|
||||||
containerId, ContainerState.RUNNING, "", 0);
|
containerId, ContainerState.RUNNING, "", 0);
|
||||||
|
|
||||||
ContainerStatus abortedStatus = BuilderUtils.newContainerStatus(
|
ContainerStatus abortedStatus = ContainerStatus.newInstance(
|
||||||
containerId, ContainerState.RUNNING, "",
|
containerId, ContainerState.RUNNING, "",
|
||||||
ContainerExitStatus.ABORTED);
|
ContainerExitStatus.ABORTED);
|
||||||
|
|
||||||
|
|
|
@ -52,6 +52,7 @@ import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.yarn.YarnException;
|
import org.apache.hadoop.yarn.YarnException;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.service.AbstractService;
|
import org.apache.hadoop.yarn.service.AbstractService;
|
||||||
|
@ -219,7 +220,7 @@ import org.junit.Test;
|
||||||
|
|
||||||
public TestMRApp(ApplicationAttemptId applicationAttemptId,
|
public TestMRApp(ApplicationAttemptId applicationAttemptId,
|
||||||
ContainerAllocator allocator, int maxAppAttempts) {
|
ContainerAllocator allocator, int maxAppAttempts) {
|
||||||
super(applicationAttemptId, BuilderUtils.newContainerId(
|
super(applicationAttemptId, ContainerId.newInstance(
|
||||||
applicationAttemptId, 1), "testhost", 2222, 3333,
|
applicationAttemptId, 1), "testhost", 2222, 3333,
|
||||||
System.currentTimeMillis(), maxAppAttempts);
|
System.currentTimeMillis(), maxAppAttempts);
|
||||||
this.allocator = allocator;
|
this.allocator = allocator;
|
||||||
|
|
|
@ -198,8 +198,8 @@ public class TestTaskAttempt{
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
|
conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
|
||||||
conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
|
conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
|
||||||
app.setClusterInfo(new ClusterInfo(BuilderUtils
|
app.setClusterInfo(new ClusterInfo(Resource
|
||||||
.newResource(minContainerSize, 1), BuilderUtils.newResource(10240, 1)));
|
.newInstance(minContainerSize, 1), Resource.newInstance(10240, 1)));
|
||||||
|
|
||||||
Job job = app.submit(conf);
|
Job job = app.submit(conf);
|
||||||
app.waitForState(job, JobState.RUNNING);
|
app.waitForState(job, JobState.RUNNING);
|
||||||
|
@ -320,7 +320,7 @@ public class TestTaskAttempt{
|
||||||
public void testLaunchFailedWhileKilling() throws Exception {
|
public void testLaunchFailedWhileKilling() throws Exception {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
BuilderUtils.newApplicationAttemptId(appId, 0);
|
ApplicationAttemptId.newInstance(appId, 0);
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
||||||
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
||||||
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
||||||
|
@ -345,8 +345,8 @@ public class TestTaskAttempt{
|
||||||
mock(Token.class), new Credentials(),
|
mock(Token.class), new Credentials(),
|
||||||
new SystemClock(), null);
|
new SystemClock(), null);
|
||||||
|
|
||||||
NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
|
NodeId nid = NodeId.newInstance("127.0.0.1", 0);
|
||||||
ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
|
ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
|
||||||
Container container = mock(Container.class);
|
Container container = mock(Container.class);
|
||||||
when(container.getId()).thenReturn(contId);
|
when(container.getId()).thenReturn(contId);
|
||||||
when(container.getNodeId()).thenReturn(nid);
|
when(container.getNodeId()).thenReturn(nid);
|
||||||
|
@ -370,7 +370,7 @@ public class TestTaskAttempt{
|
||||||
public void testContainerCleanedWhileRunning() throws Exception {
|
public void testContainerCleanedWhileRunning() throws Exception {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
BuilderUtils.newApplicationAttemptId(appId, 0);
|
ApplicationAttemptId.newInstance(appId, 0);
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
||||||
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
||||||
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
||||||
|
@ -402,8 +402,8 @@ public class TestTaskAttempt{
|
||||||
mock(Token.class), new Credentials(),
|
mock(Token.class), new Credentials(),
|
||||||
new SystemClock(), appCtx);
|
new SystemClock(), appCtx);
|
||||||
|
|
||||||
NodeId nid = BuilderUtils.newNodeId("127.0.0.2", 0);
|
NodeId nid = NodeId.newInstance("127.0.0.2", 0);
|
||||||
ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
|
ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
|
||||||
Container container = mock(Container.class);
|
Container container = mock(Container.class);
|
||||||
when(container.getId()).thenReturn(contId);
|
when(container.getId()).thenReturn(contId);
|
||||||
when(container.getNodeId()).thenReturn(nid);
|
when(container.getNodeId()).thenReturn(nid);
|
||||||
|
@ -428,7 +428,7 @@ public class TestTaskAttempt{
|
||||||
public void testContainerCleanedWhileCommitting() throws Exception {
|
public void testContainerCleanedWhileCommitting() throws Exception {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
BuilderUtils.newApplicationAttemptId(appId, 0);
|
ApplicationAttemptId.newInstance(appId, 0);
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
||||||
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
||||||
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
||||||
|
@ -460,8 +460,8 @@ public class TestTaskAttempt{
|
||||||
mock(Token.class), new Credentials(),
|
mock(Token.class), new Credentials(),
|
||||||
new SystemClock(), appCtx);
|
new SystemClock(), appCtx);
|
||||||
|
|
||||||
NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
|
NodeId nid = NodeId.newInstance("127.0.0.1", 0);
|
||||||
ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
|
ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
|
||||||
Container container = mock(Container.class);
|
Container container = mock(Container.class);
|
||||||
when(container.getId()).thenReturn(contId);
|
when(container.getId()).thenReturn(contId);
|
||||||
when(container.getNodeId()).thenReturn(nid);
|
when(container.getNodeId()).thenReturn(nid);
|
||||||
|
@ -489,7 +489,7 @@ public class TestTaskAttempt{
|
||||||
public void testDoubleTooManyFetchFailure() throws Exception {
|
public void testDoubleTooManyFetchFailure() throws Exception {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
BuilderUtils.newApplicationAttemptId(appId, 0);
|
ApplicationAttemptId.newInstance(appId, 0);
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
||||||
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
||||||
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
||||||
|
@ -521,8 +521,8 @@ public class TestTaskAttempt{
|
||||||
mock(Token.class), new Credentials(),
|
mock(Token.class), new Credentials(),
|
||||||
new SystemClock(), appCtx);
|
new SystemClock(), appCtx);
|
||||||
|
|
||||||
NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
|
NodeId nid = NodeId.newInstance("127.0.0.1", 0);
|
||||||
ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
|
ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
|
||||||
Container container = mock(Container.class);
|
Container container = mock(Container.class);
|
||||||
when(container.getId()).thenReturn(contId);
|
when(container.getId()).thenReturn(contId);
|
||||||
when(container.getNodeId()).thenReturn(nid);
|
when(container.getNodeId()).thenReturn(nid);
|
||||||
|
@ -555,7 +555,7 @@ public class TestTaskAttempt{
|
||||||
@Test
|
@Test
|
||||||
public void testAppDiognosticEventOnUnassignedTask() throws Exception {
|
public void testAppDiognosticEventOnUnassignedTask() throws Exception {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
||||||
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
|
||||||
appId, 0);
|
appId, 0);
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
||||||
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
||||||
|
@ -587,8 +587,8 @@ public class TestTaskAttempt{
|
||||||
jobFile, 1, splits, jobConf, taListener,
|
jobFile, 1, splits, jobConf, taListener,
|
||||||
mock(Token.class), new Credentials(), new SystemClock(), appCtx);
|
mock(Token.class), new Credentials(), new SystemClock(), appCtx);
|
||||||
|
|
||||||
NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
|
NodeId nid = NodeId.newInstance("127.0.0.1", 0);
|
||||||
ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
|
ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
|
||||||
Container container = mock(Container.class);
|
Container container = mock(Container.class);
|
||||||
when(container.getId()).thenReturn(contId);
|
when(container.getId()).thenReturn(contId);
|
||||||
when(container.getNodeId()).thenReturn(nid);
|
when(container.getNodeId()).thenReturn(nid);
|
||||||
|
@ -605,7 +605,7 @@ public class TestTaskAttempt{
|
||||||
@Test
|
@Test
|
||||||
public void testAppDiognosticEventOnNewTask() throws Exception {
|
public void testAppDiognosticEventOnNewTask() throws Exception {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
ApplicationId appId = BuilderUtils.newApplicationId(1, 2);
|
||||||
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
|
||||||
appId, 0);
|
appId, 0);
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
|
||||||
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
|
||||||
|
@ -637,8 +637,8 @@ public class TestTaskAttempt{
|
||||||
jobFile, 1, splits, jobConf, taListener,
|
jobFile, 1, splits, jobConf, taListener,
|
||||||
mock(Token.class), new Credentials(), new SystemClock(), appCtx);
|
mock(Token.class), new Credentials(), new SystemClock(), appCtx);
|
||||||
|
|
||||||
NodeId nid = BuilderUtils.newNodeId("127.0.0.1", 0);
|
NodeId nid = NodeId.newInstance("127.0.0.1", 0);
|
||||||
ContainerId contId = BuilderUtils.newContainerId(appAttemptId, 3);
|
ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
|
||||||
Container container = mock(Container.class);
|
Container container = mock(Container.class);
|
||||||
when(container.getId()).thenReturn(contId);
|
when(container.getId()).thenReturn(contId);
|
||||||
when(container.getNodeId()).thenReturn(nid);
|
when(container.getNodeId()).thenReturn(nid);
|
||||||
|
|
|
@ -83,7 +83,7 @@ public class TestContainerLauncher {
|
||||||
public void testPoolSize() throws InterruptedException {
|
public void testPoolSize() throws InterruptedException {
|
||||||
|
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(12345, 67);
|
ApplicationId appId = BuilderUtils.newApplicationId(12345, 67);
|
||||||
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
|
||||||
appId, 3);
|
appId, 3);
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 8);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 8);
|
||||||
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 9, TaskType.MAP);
|
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 9, TaskType.MAP);
|
||||||
|
@ -104,7 +104,7 @@ public class TestContainerLauncher {
|
||||||
|
|
||||||
containerLauncher.expectedCorePoolSize = ContainerLauncherImpl.INITIAL_POOL_SIZE;
|
containerLauncher.expectedCorePoolSize = ContainerLauncherImpl.INITIAL_POOL_SIZE;
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, i);
|
ContainerId containerId = ContainerId.newInstance(appAttemptId, i);
|
||||||
TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, i);
|
TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, i);
|
||||||
containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId,
|
containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId,
|
||||||
containerId, "host" + i + ":1234", null,
|
containerId, "host" + i + ":1234", null,
|
||||||
|
@ -126,7 +126,7 @@ public class TestContainerLauncher {
|
||||||
Assert.assertEquals(10, containerLauncher.numEventsProcessed.get());
|
Assert.assertEquals(10, containerLauncher.numEventsProcessed.get());
|
||||||
containerLauncher.finishEventHandling = false;
|
containerLauncher.finishEventHandling = false;
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId,
|
ContainerId containerId = ContainerId.newInstance(appAttemptId,
|
||||||
i + 10);
|
i + 10);
|
||||||
TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId,
|
TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId,
|
||||||
i + 10);
|
i + 10);
|
||||||
|
@ -143,7 +143,7 @@ public class TestContainerLauncher {
|
||||||
// Core pool size should be 21 but the live pool size should be only 11.
|
// Core pool size should be 21 but the live pool size should be only 11.
|
||||||
containerLauncher.expectedCorePoolSize = 11 + ContainerLauncherImpl.INITIAL_POOL_SIZE;
|
containerLauncher.expectedCorePoolSize = 11 + ContainerLauncherImpl.INITIAL_POOL_SIZE;
|
||||||
containerLauncher.finishEventHandling = false;
|
containerLauncher.finishEventHandling = false;
|
||||||
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 21);
|
ContainerId containerId = ContainerId.newInstance(appAttemptId, 21);
|
||||||
TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, 21);
|
TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, 21);
|
||||||
containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId,
|
containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId,
|
||||||
containerId, "host11:1234", null,
|
containerId, "host11:1234", null,
|
||||||
|
@ -158,12 +158,12 @@ public class TestContainerLauncher {
|
||||||
@Test
|
@Test
|
||||||
public void testPoolLimits() throws InterruptedException {
|
public void testPoolLimits() throws InterruptedException {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(12345, 67);
|
ApplicationId appId = BuilderUtils.newApplicationId(12345, 67);
|
||||||
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
|
||||||
appId, 3);
|
appId, 3);
|
||||||
JobId jobId = MRBuilderUtils.newJobId(appId, 8);
|
JobId jobId = MRBuilderUtils.newJobId(appId, 8);
|
||||||
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 9, TaskType.MAP);
|
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 9, TaskType.MAP);
|
||||||
TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
|
||||||
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 10);
|
ContainerId containerId = ContainerId.newInstance(appAttemptId, 10);
|
||||||
|
|
||||||
AppContext context = mock(AppContext.class);
|
AppContext context = mock(AppContext.class);
|
||||||
CustomContainerLauncher containerLauncher = new CustomContainerLauncher(
|
CustomContainerLauncher containerLauncher = new CustomContainerLauncher(
|
||||||
|
|
|
@ -19,12 +19,11 @@ package org.apache.hadoop.mapreduce.v2.app.launcher;
|
||||||
|
|
||||||
import static org.mockito.Matchers.any;
|
import static org.mockito.Matchers.any;
|
||||||
import static org.mockito.Matchers.eq;
|
import static org.mockito.Matchers.eq;
|
||||||
|
import static org.mockito.Mockito.atLeast;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.never;
|
import static org.mockito.Mockito.never;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
import static org.mockito.Mockito.atLeast;
|
|
||||||
import org.mockito.ArgumentCaptor;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
|
@ -53,9 +52,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerToken;
|
import org.apache.hadoop.yarn.api.records.ContainerToken;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.event.Event;
|
import org.apache.hadoop.yarn.event.Event;
|
||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
|
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
|
||||||
|
@ -66,6 +68,7 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
import org.apache.hadoop.yarn.util.BuilderUtils;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
|
||||||
public class TestContainerLauncherImpl {
|
public class TestContainerLauncherImpl {
|
||||||
static final Log LOG = LogFactory.getLog(TestContainerLauncherImpl.class);
|
static final Log LOG = LogFactory.getLog(TestContainerLauncherImpl.class);
|
||||||
|
@ -122,8 +125,8 @@ public class TestContainerLauncherImpl {
|
||||||
|
|
||||||
public static ContainerId makeContainerId(long ts, int appId, int attemptId,
|
public static ContainerId makeContainerId(long ts, int appId, int attemptId,
|
||||||
int id) {
|
int id) {
|
||||||
return BuilderUtils.newContainerId(
|
return ContainerId.newInstance(
|
||||||
BuilderUtils.newApplicationAttemptId(
|
ApplicationAttemptId.newInstance(
|
||||||
BuilderUtils.newApplicationId(ts, appId), attemptId), id);
|
BuilderUtils.newApplicationId(ts, appId), attemptId), id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -406,10 +409,10 @@ public class TestContainerLauncherImpl {
|
||||||
|
|
||||||
private ContainerToken createNewContainerToken(ContainerId contId,
|
private ContainerToken createNewContainerToken(ContainerId contId,
|
||||||
String containerManagerAddr) {
|
String containerManagerAddr) {
|
||||||
return BuilderUtils.newContainerToken(BuilderUtils.newNodeId("127.0.0.1",
|
return BuilderUtils.newContainerToken(NodeId.newInstance("127.0.0.1",
|
||||||
1234), "password".getBytes(), new ContainerTokenIdentifier(
|
1234), "password".getBytes(), new ContainerTokenIdentifier(
|
||||||
contId, containerManagerAddr, "user",
|
contId, containerManagerAddr, "user",
|
||||||
BuilderUtils.newResource(1024, 1),
|
Resource.newInstance(1024, 1),
|
||||||
System.currentTimeMillis() + 10000L, 123));
|
System.currentTimeMillis() + 10000L, 123));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.AMRMProtocol;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
|
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
|
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
|
||||||
import org.apache.hadoop.yarn.ipc.RPCUtil;
|
import org.apache.hadoop.yarn.ipc.RPCUtil;
|
||||||
|
@ -108,7 +109,7 @@ public class TestLocalContainerAllocator {
|
||||||
private static AppContext createAppContext() {
|
private static AppContext createAppContext() {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
|
ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
|
||||||
ApplicationAttemptId attemptId =
|
ApplicationAttemptId attemptId =
|
||||||
BuilderUtils.newApplicationAttemptId(appId, 1);
|
ApplicationAttemptId.newInstance(appId, 1);
|
||||||
Job job = mock(Job.class);
|
Job job = mock(Job.class);
|
||||||
@SuppressWarnings("rawtypes")
|
@SuppressWarnings("rawtypes")
|
||||||
EventHandler eventHandler = mock(EventHandler.class);
|
EventHandler eventHandler = mock(EventHandler.class);
|
||||||
|
@ -117,8 +118,8 @@ public class TestLocalContainerAllocator {
|
||||||
when(ctx.getApplicationAttemptId()).thenReturn(attemptId);
|
when(ctx.getApplicationAttemptId()).thenReturn(attemptId);
|
||||||
when(ctx.getJob(isA(JobId.class))).thenReturn(job);
|
when(ctx.getJob(isA(JobId.class))).thenReturn(job);
|
||||||
when(ctx.getClusterInfo()).thenReturn(
|
when(ctx.getClusterInfo()).thenReturn(
|
||||||
new ClusterInfo(BuilderUtils.newResource(1024, 1), BuilderUtils
|
new ClusterInfo(Resource.newInstance(1024, 1), Resource.newInstance(
|
||||||
.newResource(10240, 1)));
|
10240, 1)));
|
||||||
when(ctx.getEventHandler()).thenReturn(eventHandler);
|
when(ctx.getEventHandler()).thenReturn(eventHandler);
|
||||||
return ctx;
|
return ctx;
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,8 +46,8 @@ import org.apache.hadoop.yarn.Clock;
|
||||||
import org.apache.hadoop.yarn.ClusterInfo;
|
import org.apache.hadoop.yarn.ClusterInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
import org.apache.hadoop.yarn.util.Times;
|
import org.apache.hadoop.yarn.util.Times;
|
||||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||||
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
|
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
|
||||||
|
@ -66,9 +66,9 @@ import com.google.inject.Injector;
|
||||||
import com.google.inject.servlet.GuiceServletContextListener;
|
import com.google.inject.servlet.GuiceServletContextListener;
|
||||||
import com.google.inject.servlet.ServletModule;
|
import com.google.inject.servlet.ServletModule;
|
||||||
import com.sun.jersey.api.client.ClientResponse;
|
import com.sun.jersey.api.client.ClientResponse;
|
||||||
|
import com.sun.jersey.api.client.ClientResponse.Status;
|
||||||
import com.sun.jersey.api.client.UniformInterfaceException;
|
import com.sun.jersey.api.client.UniformInterfaceException;
|
||||||
import com.sun.jersey.api.client.WebResource;
|
import com.sun.jersey.api.client.WebResource;
|
||||||
import com.sun.jersey.api.client.ClientResponse.Status;
|
|
||||||
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
||||||
import com.sun.jersey.test.framework.JerseyTest;
|
import com.sun.jersey.test.framework.JerseyTest;
|
||||||
import com.sun.jersey.test.framework.WebAppDescriptor;
|
import com.sun.jersey.test.framework.WebAppDescriptor;
|
||||||
|
@ -967,7 +967,7 @@ public class TestAMWebServicesJobs extends JerseyTest {
|
||||||
WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":"
|
WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":"
|
||||||
+ nmHttpPort, nodeHttpAddress);
|
+ nmHttpPort, nodeHttpAddress);
|
||||||
WebServicesTestUtils.checkStringMatch("nodeId",
|
WebServicesTestUtils.checkStringMatch("nodeId",
|
||||||
BuilderUtils.newNodeId(nmHost, nmPort).toString(), nodeId);
|
NodeId.newInstance(nmHost, nmPort).toString(), nodeId);
|
||||||
assertTrue("startime not greater than 0", startTime > 0);
|
assertTrue("startime not greater than 0", startTime > 0);
|
||||||
WebServicesTestUtils.checkStringMatch("containerId", amInfo
|
WebServicesTestUtils.checkStringMatch("containerId", amInfo
|
||||||
.getContainerId().toString(), containerId);
|
.getContainerId().toString(), containerId);
|
||||||
|
|
|
@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.util.ApplicationClassLoader;
|
import org.apache.hadoop.yarn.util.ApplicationClassLoader;
|
||||||
import org.apache.hadoop.yarn.util.Apps;
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -435,15 +434,10 @@ public class MRApps extends Apps {
|
||||||
" This will be an error in Hadoop 2.0");
|
" This will be an error in Hadoop 2.0");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
localResources.put(
|
localResources.put(linkName, LocalResource.newInstance(ConverterUtils
|
||||||
linkName,
|
.getYarnUrlFromURI(p.toUri()), type, visibilities[i]
|
||||||
BuilderUtils.newLocalResource(
|
? LocalResourceVisibility.PUBLIC : LocalResourceVisibility.PRIVATE,
|
||||||
p.toUri(), type,
|
sizes[i], timestamps[i]));
|
||||||
visibilities[i]
|
|
||||||
? LocalResourceVisibility.PUBLIC
|
|
||||||
: LocalResourceVisibility.PRIVATE,
|
|
||||||
sizes[i], timestamps[i])
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,7 +28,6 @@ import javax.xml.bind.annotation.XmlTransient;
|
||||||
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
|
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
|
|
||||||
@XmlRootElement(name = "jobAttempt")
|
@XmlRootElement(name = "jobAttempt")
|
||||||
@XmlAccessorType(XmlAccessType.FIELD)
|
@XmlAccessorType(XmlAccessType.FIELD)
|
||||||
|
@ -56,7 +55,7 @@ public class AMAttemptInfo {
|
||||||
int nmPort = amInfo.getNodeManagerPort();
|
int nmPort = amInfo.getNodeManagerPort();
|
||||||
if (nmHost != null) {
|
if (nmHost != null) {
|
||||||
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
|
this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
|
||||||
NodeId nodeId = BuilderUtils.newNodeId(nmHost, nmPort);
|
NodeId nodeId = NodeId.newInstance(nmHost, nmPort);
|
||||||
this.nodeId = nodeId.toString();
|
this.nodeId = nodeId.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.Clock;
|
||||||
import org.apache.hadoop.yarn.ClusterInfo;
|
import org.apache.hadoop.yarn.ClusterInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
import org.apache.hadoop.yarn.util.BuilderUtils;
|
||||||
|
@ -243,7 +244,7 @@ public class TestHSWebApp {
|
||||||
params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
|
params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
|
||||||
.toString());
|
.toString());
|
||||||
params.put(NM_NODENAME,
|
params.put(NM_NODENAME,
|
||||||
BuilderUtils.newNodeId(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
|
NodeId.newInstance(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
|
||||||
params.put(ENTITY_STRING, "container_10_0001_01_000001");
|
params.put(ENTITY_STRING, "container_10_0001_01_000001");
|
||||||
params.put(APP_OWNER, "owner");
|
params.put(APP_OWNER, "owner");
|
||||||
|
|
||||||
|
@ -271,7 +272,7 @@ public class TestHSWebApp {
|
||||||
params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
|
params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
|
||||||
.toString());
|
.toString());
|
||||||
params.put(NM_NODENAME,
|
params.put(NM_NODENAME,
|
||||||
BuilderUtils.newNodeId(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
|
NodeId.newInstance(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
|
||||||
params.put(ENTITY_STRING, "container_10_0001_01_000001");
|
params.put(ENTITY_STRING, "container_10_0001_01_000001");
|
||||||
params.put(APP_OWNER, "owner");
|
params.put(APP_OWNER, "owner");
|
||||||
|
|
||||||
|
@ -302,7 +303,7 @@ public class TestHSWebApp {
|
||||||
params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
|
params.put(CONTAINER_ID, BuilderUtils.newContainerId(1, 1, 333, 1)
|
||||||
.toString());
|
.toString());
|
||||||
params.put(NM_NODENAME,
|
params.put(NM_NODENAME,
|
||||||
BuilderUtils.newNodeId(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
|
NodeId.newInstance(MockJobs.NM_HOST, MockJobs.NM_PORT).toString());
|
||||||
params.put(ENTITY_STRING, "container_10_0001_01_000001");
|
params.put(ENTITY_STRING, "container_10_0001_01_000001");
|
||||||
params.put(APP_OWNER, "owner");
|
params.put(APP_OWNER, "owner");
|
||||||
|
|
||||||
|
|
|
@ -53,8 +53,8 @@ import org.apache.hadoop.yarn.ClusterInfo;
|
||||||
import org.apache.hadoop.yarn.YarnException;
|
import org.apache.hadoop.yarn.YarnException;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||||
import org.apache.hadoop.yarn.webapp.WebApp;
|
import org.apache.hadoop.yarn.webapp.WebApp;
|
||||||
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
|
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
|
||||||
|
@ -913,7 +913,7 @@ public class TestHsWebServicesJobs extends JerseyTest {
|
||||||
WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":"
|
WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":"
|
||||||
+ nmHttpPort, nodeHttpAddress);
|
+ nmHttpPort, nodeHttpAddress);
|
||||||
WebServicesTestUtils.checkStringMatch("nodeId",
|
WebServicesTestUtils.checkStringMatch("nodeId",
|
||||||
BuilderUtils.newNodeId(nmHost, nmPort).toString(), nodeId);
|
NodeId.newInstance(nmHost, nmPort).toString(), nodeId);
|
||||||
assertTrue("startime not greater than 0", startTime > 0);
|
assertTrue("startime not greater than 0", startTime > 0);
|
||||||
WebServicesTestUtils.checkStringMatch("containerId", amInfo
|
WebServicesTestUtils.checkStringMatch("containerId", amInfo
|
||||||
.getContainerId().toString(), containerId);
|
.getContainerId().toString(), containerId);
|
||||||
|
|
|
@ -71,6 +71,7 @@ import org.apache.hadoop.yarn.YarnException;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ClientToken;
|
import org.apache.hadoop.yarn.api.records.ClientToken;
|
||||||
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
|
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
|
@ -478,7 +479,7 @@ public class ClientServiceDelegate {
|
||||||
taReport.getContainerId().toString(),
|
taReport.getContainerId().toString(),
|
||||||
taReport.getContainerId().getApplicationAttemptId()
|
taReport.getContainerId().getApplicationAttemptId()
|
||||||
.getApplicationId().toString(),
|
.getApplicationId().toString(),
|
||||||
BuilderUtils.newNodeId(taReport.getNodeManagerHost(),
|
NodeId.newInstance(taReport.getNodeManagerHost(),
|
||||||
taReport.getNodeManagerPort()).toString(), report.getUser());
|
taReport.getNodeManagerPort()).toString(), report.getUser());
|
||||||
} else {
|
} else {
|
||||||
if (report.getAMInfos() == null || report.getAMInfos().size() == 0) {
|
if (report.getAMInfos() == null || report.getAMInfos().size() == 0) {
|
||||||
|
@ -489,7 +490,7 @@ public class ClientServiceDelegate {
|
||||||
return new LogParams(
|
return new LogParams(
|
||||||
amInfo.getContainerId().toString(),
|
amInfo.getContainerId().toString(),
|
||||||
amInfo.getAppAttemptId().getApplicationId().toString(),
|
amInfo.getAppAttemptId().getApplicationId().toString(),
|
||||||
BuilderUtils.newNodeId(amInfo.getNodeManagerHost(),
|
NodeId.newInstance(amInfo.getNodeManagerHost(),
|
||||||
amInfo.getNodeManagerPort()).toString(), report.getUser());
|
amInfo.getNodeManagerPort()).toString(), report.getUser());
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -88,7 +88,7 @@ public class NotRunningJob implements MRClientProtocol {
|
||||||
|
|
||||||
// Setting AppState to NEW and finalStatus to UNDEFINED as they are never
|
// Setting AppState to NEW and finalStatus to UNDEFINED as they are never
|
||||||
// used for a non running job
|
// used for a non running job
|
||||||
return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId,
|
return ApplicationReport.newInstance(unknownAppId, unknownAttemptId,
|
||||||
"N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
|
"N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
|
||||||
"N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
|
"N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
|
||||||
YarnConfiguration.DEFAULT_APPLICATION_TYPE);
|
YarnConfiguration.DEFAULT_APPLICATION_TYPE);
|
||||||
|
|
|
@ -84,7 +84,6 @@ import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.security.client.RMTokenSelector;
|
import org.apache.hadoop.yarn.security.client.RMTokenSelector;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ProtoUtils;
|
||||||
|
|
||||||
|
@ -475,9 +474,10 @@ public class YARNRunner implements ClientProtocol {
|
||||||
MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));
|
MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));
|
||||||
|
|
||||||
// Setup ContainerLaunchContext for AM container
|
// Setup ContainerLaunchContext for AM container
|
||||||
ContainerLaunchContext amContainer = BuilderUtils
|
ContainerLaunchContext amContainer =
|
||||||
.newContainerLaunchContext(localResources,
|
ContainerLaunchContext.newInstance(localResources, environment,
|
||||||
environment, vargsFinal, null, securityTokens, acls);
|
vargsFinal, null, securityTokens, acls);
|
||||||
|
|
||||||
|
|
||||||
// Set up the ApplicationSubmissionContext
|
// Set up the ApplicationSubmissionContext
|
||||||
ApplicationSubmissionContext appContext =
|
ApplicationSubmissionContext appContext =
|
||||||
|
|
|
@ -426,9 +426,9 @@ public class TestClientServiceDelegate {
|
||||||
|
|
||||||
private ApplicationReport getFinishedApplicationReport() {
|
private ApplicationReport getFinishedApplicationReport() {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1234, 5);
|
ApplicationId appId = BuilderUtils.newApplicationId(1234, 5);
|
||||||
ApplicationAttemptId attemptId = BuilderUtils.newApplicationAttemptId(
|
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
|
||||||
appId, 0);
|
appId, 0);
|
||||||
return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue",
|
return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
|
||||||
"appname", "host", 124, null, YarnApplicationState.FINISHED,
|
"appname", "host", 124, null, YarnApplicationState.FINISHED,
|
||||||
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
|
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
|
||||||
"N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE);
|
"N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE);
|
||||||
|
@ -436,9 +436,9 @@ public class TestClientServiceDelegate {
|
||||||
|
|
||||||
private ApplicationReport getRunningApplicationReport(String host, int port) {
|
private ApplicationReport getRunningApplicationReport(String host, int port) {
|
||||||
ApplicationId appId = BuilderUtils.newApplicationId(1234, 5);
|
ApplicationId appId = BuilderUtils.newApplicationId(1234, 5);
|
||||||
ApplicationAttemptId attemptId = BuilderUtils.newApplicationAttemptId(
|
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
|
||||||
appId, 0);
|
appId, 0);
|
||||||
return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue",
|
return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
|
||||||
"appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics",
|
"appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics",
|
||||||
"url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
|
"url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
|
||||||
YarnConfiguration.DEFAULT_APPLICATION_TYPE);
|
YarnConfiguration.DEFAULT_APPLICATION_TYPE);
|
||||||
|
|
|
@ -48,10 +48,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
public class TestMRJobsWithHistoryService {
|
public class TestMRJobsWithHistoryService {
|
||||||
|
@ -169,8 +168,8 @@ public class TestMRJobsWithHistoryService {
|
||||||
List<AMInfo> amInfos = jobReport.getAMInfos();
|
List<AMInfo> amInfos = jobReport.getAMInfos();
|
||||||
Assert.assertEquals(1, amInfos.size());
|
Assert.assertEquals(1, amInfos.size());
|
||||||
AMInfo amInfo = amInfos.get(0);
|
AMInfo amInfo = amInfos.get(0);
|
||||||
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(jobId.getAppId(), 1);
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(jobId.getAppId(), 1);
|
||||||
ContainerId amContainerId = BuilderUtils.newContainerId(appAttemptId, 1);
|
ContainerId amContainerId = ContainerId.newInstance(appAttemptId, 1);
|
||||||
Assert.assertEquals(appAttemptId, amInfo.getAppAttemptId());
|
Assert.assertEquals(appAttemptId, amInfo.getAppAttemptId());
|
||||||
Assert.assertEquals(amContainerId, amInfo.getContainerId());
|
Assert.assertEquals(amContainerId, amInfo.getContainerId());
|
||||||
Assert.assertTrue(jobReport.getSubmitTime() > 0);
|
Assert.assertTrue(jobReport.getSubmitTime() > 0);
|
||||||
|
|
Loading…
Reference in New Issue