Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6411. Clean up the overwrite of createDispatcher() in subclass of MockRM. Contributed by Yufei Gu 

(cherry picked from commit 4d1fac5df2)

Conflicts:
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
This commit is contained in:
Jason Lowe 2017-03-31 10:05:34 -05:00
parent 9e7f8cbf9a
commit 3ae529ba54
15 changed files with 277 additions and 475 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

453
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
View File

File diff suppressed because it is too large Load Diff

59
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
View File

@ -48,8 +48,6 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@ -125,22 +123,20 @@ public class TestAMRMClientOnRMRestart {
// Phase-1 Start 1st RM // Phase-1 Start 1st RM
MyResourceManager rm1 = new MyResourceManager(conf, memStore); MyResourceManager rm1 = new MyResourceManager(conf, memStore);
rm1.start(); rm1.start();
DrainDispatcher dispatcher =
(DrainDispatcher) rm1.getRMContext().getDispatcher();
// Submit the application // Submit the application
RMApp app = rm1.submitApp(1024); RMApp app = rm1.submitApp(1024);
dispatcher.await(); rm1.drainEvents();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode(); nm1.registerNode();
nm1.nodeHeartbeat(true); // Node heartbeat nm1.nodeHeartbeat(true); // Node heartbeat
dispatcher.await(); rm1.drainEvents();
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId(); app.getCurrentAppAttempt().getAppAttemptId();
rm1.sendAMLaunched(appAttemptId); rm1.sendAMLaunched(appAttemptId);
dispatcher.await(); rm1.drainEvents();
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token =
rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId()) rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
@ -175,7 +171,7 @@ public class TestAMRMClientOnRMRestart {
blacklistAdditions.remove("h2");// remove from local list blacklistAdditions.remove("h2");// remove from local list
AllocateResponse allocateResponse = amClient.allocate(0.1f); AllocateResponse allocateResponse = amClient.allocate(0.1f);
dispatcher.await(); rm1.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
.getAllocatedContainers().size()); .getAllocatedContainers().size());
@ -188,10 +184,10 @@ public class TestAMRMClientOnRMRestart {
// Step-2 : NM heart beat is sent. // Step-2 : NM heart beat is sent.
// On 2nd AM allocate request, RM allocates 3 containers to AM // On 2nd AM allocate request, RM allocates 3 containers to AM
nm1.nodeHeartbeat(true); // Node heartbeat nm1.nodeHeartbeat(true); // Node heartbeat
dispatcher.await(); rm1.drainEvents();
allocateResponse = amClient.allocate(0.2f); allocateResponse = amClient.allocate(0.2f);
dispatcher.await(); rm1.drainEvents();
// 3 containers are allocated i.e for cRequest1, cRequest2 and cRequest3. // 3 containers are allocated i.e for cRequest1, cRequest2 and cRequest3.
Assert.assertEquals("No of assignments must be 0", 3, allocateResponse Assert.assertEquals("No of assignments must be 0", 3, allocateResponse
.getAllocatedContainers().size()); .getAllocatedContainers().size());
@ -206,7 +202,7 @@ public class TestAMRMClientOnRMRestart {
amClient.removeContainerRequest(cRequest3); amClient.removeContainerRequest(cRequest3);
allocateResponse = amClient.allocate(0.2f); allocateResponse = amClient.allocate(0.2f);
dispatcher.await(); rm1.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
.getAllocatedContainers().size()); .getAllocatedContainers().size());
assertAsksAndReleases(4, 0, rm1); assertAsksAndReleases(4, 0, rm1);
@ -232,13 +228,13 @@ public class TestAMRMClientOnRMRestart {
// request // request
nm1.nodeHeartbeat(containerId.getApplicationAttemptId(), nm1.nodeHeartbeat(containerId.getApplicationAttemptId(),
containerId.getContainerId(), ContainerState.RUNNING); containerId.getContainerId(), ContainerState.RUNNING);
dispatcher.await(); rm1.drainEvents();
amClient.requestContainerResourceChange( amClient.requestContainerResourceChange(
container, Resource.newInstance(2048, 1)); container, Resource.newInstance(2048, 1));
it.remove(); it.remove();
allocateResponse = amClient.allocate(0.3f); allocateResponse = amClient.allocate(0.3f);
dispatcher.await(); rm1.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
.getAllocatedContainers().size()); .getAllocatedContainers().size());
assertAsksAndReleases(3, pendingRelease, rm1); assertAsksAndReleases(3, pendingRelease, rm1);
@ -254,7 +250,6 @@ public class TestAMRMClientOnRMRestart {
rm2.start(); rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService()); nm1.setResourceTrackerService(rm2.getResourceTrackerService());
((MyAMRMClientImpl) amClient).updateRMProxy(rm2); ((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
// NM should be rebooted on heartbeat, even first heartbeat for nm2 // NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true); NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
@ -270,7 +265,7 @@ public class TestAMRMClientOnRMRestart {
Collections.singletonList( Collections.singletonList(
containerId.getApplicationAttemptId().getApplicationId())); containerId.getApplicationAttemptId().getApplicationId()));
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm2.drainEvents();
blacklistAdditions.add("h3"); blacklistAdditions.add("h3");
amClient.updateBlacklist(blacklistAdditions, null); amClient.updateBlacklist(blacklistAdditions, null);
@ -292,7 +287,7 @@ public class TestAMRMClientOnRMRestart {
// containerRequest and blacklisted nodes. // containerRequest and blacklisted nodes.
// Intern RM send resync command,AMRMClient resend allocate request // Intern RM send resync command,AMRMClient resend allocate request
allocateResponse = amClient.allocate(0.3f); allocateResponse = amClient.allocate(0.3f);
dispatcher.await(); rm2.drainEvents();
completedContainer = completedContainer =
allocateResponse.getCompletedContainersStatuses().size(); allocateResponse.getCompletedContainersStatuses().size();
@ -309,7 +304,7 @@ public class TestAMRMClientOnRMRestart {
// Step-5 : Allocater after resync command // Step-5 : Allocater after resync command
allocateResponse = amClient.allocate(0.5f); allocateResponse = amClient.allocate(0.5f);
dispatcher.await(); rm2.drainEvents();
Assert.assertEquals("No of assignments must be 0", 0, allocateResponse Assert.assertEquals("No of assignments must be 0", 0, allocateResponse
.getAllocatedContainers().size()); .getAllocatedContainers().size());
@ -322,10 +317,10 @@ public class TestAMRMClientOnRMRestart {
int count = 5; int count = 5;
while (count-- > 0) { while (count-- > 0) {
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm2.drainEvents();
allocateResponse = amClient.allocate(0.5f); allocateResponse = amClient.allocate(0.5f);
dispatcher.await(); rm2.drainEvents();
noAssignedContainer += allocateResponse.getAllocatedContainers().size(); noAssignedContainer += allocateResponse.getAllocatedContainers().size();
if (noAssignedContainer == 3) { if (noAssignedContainer == 3) {
break; break;
@ -354,22 +349,20 @@ public class TestAMRMClientOnRMRestart {
// Phase-1 Start 1st RM // Phase-1 Start 1st RM
MyResourceManager rm1 = new MyResourceManager(conf, memStore); MyResourceManager rm1 = new MyResourceManager(conf, memStore);
rm1.start(); rm1.start();
DrainDispatcher dispatcher =
(DrainDispatcher) rm1.getRMContext().getDispatcher();
// Submit the application // Submit the application
RMApp app = rm1.submitApp(1024); RMApp app = rm1.submitApp(1024);
dispatcher.await(); rm1.drainEvents();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode(); nm1.registerNode();
nm1.nodeHeartbeat(true); // Node heartbeat nm1.nodeHeartbeat(true); // Node heartbeat
dispatcher.await(); rm1.drainEvents();
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId(); app.getCurrentAppAttempt().getAppAttemptId();
rm1.sendAMLaunched(appAttemptId); rm1.sendAMLaunched(appAttemptId);
dispatcher.await(); rm1.drainEvents();
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token =
rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId()) rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId())
@ -389,7 +382,6 @@ public class TestAMRMClientOnRMRestart {
rm2.start(); rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService()); nm1.setResourceTrackerService(rm2.getResourceTrackerService());
((MyAMRMClientImpl) amClient).updateRMProxy(rm2); ((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
// NM should be rebooted on heartbeat, even first heartbeat for nm2 // NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true); NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
@ -405,7 +397,7 @@ public class TestAMRMClientOnRMRestart {
Priority.newInstance(0), 0); Priority.newInstance(0), 0);
nm1.registerNode(Arrays.asList(containerReport), null); nm1.registerNode(Arrays.asList(containerReport), null);
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm2.drainEvents();
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null); null, null);
@ -417,7 +409,6 @@ public class TestAMRMClientOnRMRestart {
amClient.stop(); amClient.stop();
rm1.stop(); rm1.stop();
rm2.stop(); rm2.stop();
} }
@ -435,22 +426,20 @@ public class TestAMRMClientOnRMRestart {
// start first RM // start first RM
MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore); MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore);
rm1.start(); rm1.start();
DrainDispatcher dispatcher =
(DrainDispatcher) rm1.getRMContext().getDispatcher();
Long startTime = System.currentTimeMillis(); Long startTime = System.currentTimeMillis();
// Submit the application // Submit the application
RMApp app = rm1.submitApp(1024); RMApp app = rm1.submitApp(1024);
dispatcher.await(); rm1.drainEvents();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode(); nm1.registerNode();
nm1.nodeHeartbeat(true); // Node heartbeat nm1.nodeHeartbeat(true); // Node heartbeat
dispatcher.await(); rm1.drainEvents();
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
app.getCurrentAppAttempt().getAppAttemptId(); app.getCurrentAppAttempt().getAppAttemptId();
rm1.sendAMLaunched(appAttemptId); rm1.sendAMLaunched(appAttemptId);
dispatcher.await(); rm1.drainEvents();
AMRMTokenSecretManager amrmTokenSecretManagerForRM1 = AMRMTokenSecretManager amrmTokenSecretManagerForRM1 =
rm1.getRMContext().getAMRMTokenSecretManager(); rm1.getRMContext().getAMRMTokenSecretManager();
@ -509,7 +498,6 @@ public class TestAMRMClientOnRMRestart {
rm2.start(); rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService()); nm1.setResourceTrackerService(rm2.getResourceTrackerService());
((MyAMRMClientImpl) amClient).updateRMProxy(rm2); ((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
AMRMTokenSecretManager amrmTokenSecretManagerForRM2 = AMRMTokenSecretManager amrmTokenSecretManagerForRM2 =
rm2.getRMContext().getAMRMTokenSecretManager(); rm2.getRMContext().getAMRMTokenSecretManager();
@ -611,11 +599,6 @@ public class TestAMRMClientOnRMRestart {
MyResourceManager.setClusterTimeStamp(fakeClusterTimeStamp); MyResourceManager.setClusterTimeStamp(fakeClusterTimeStamp);
} }
@Override
protected Dispatcher createDispatcher() {
return new DrainDispatcher();
}
@Override @Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() { protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
// Dispatch inline for test sanity // Dispatch inline for test sanity

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
View File

@ -30,14 +30,9 @@ import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.junit.Before; import org.junit.Before;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public abstract class ACLsTestBase { public abstract class ACLsTestBase {
protected static final String COMMON_USER = "common_user"; protected static final String COMMON_USER = "common_user";
@ -80,11 +75,6 @@ public abstract class ACLsTestBase {
.getRMDelegationTokenSecretManager()); .getRMDelegationTokenSecretManager());
} }
@Override
protected Dispatcher createDispatcher() {
return new DrainDispatcher();
}
@Override @Override
protected void doSecureLogin() throws IOException { protected void doSecureLogin() throws IOException {
} }

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
View File

@ -26,22 +26,17 @@ import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -108,20 +103,9 @@ public abstract class RMHATestBase extends ClientBaseWithFixes{
} }
protected void startRMs() throws IOException { protected void startRMs() throws IOException {
rm1 = new MockRM(confForRM1, null, false, false){ rm1 = new MockRM(confForRM1, null, false, false);
@Override rm2 = new MockRM(confForRM2, null, false, false);
protected Dispatcher createDispatcher() {
return new DrainDispatcher();
}
};
rm2 = new MockRM(confForRM2, null, false, false){
@Override
protected Dispatcher createDispatcher() {
return new DrainDispatcher();
}
};
startRMs(rm1, confForRM1, rm2, confForRM2); startRMs(rm1, confForRM1, rm2, confForRM2);
} }
protected void startRMsWithCustomizedRMAppManager() throws IOException { protected void startRMsWithCustomizedRMAppManager() throws IOException {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
View File

@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.api.records.ReservationRequest;
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.ReservationRequests;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@ -463,9 +462,7 @@ public class ReservationACLsTestBase extends ACLsTestBase {
int attempts = 10; int attempts = 10;
Collection<Plan> plans; Collection<Plan> plans;
do { do {
DrainDispatcher dispatcher = resourceManager.drainEvents();
(DrainDispatcher) resourceManager.getRMContext().getDispatcher();
dispatcher.await();
LOG.info("Waiting for node capacity to be added to plan"); LOG.info("Waiting for node capacity to be added to plan");
plans = resourceManager.getRMContext().getReservationSystem() plans = resourceManager.getRMContext().getReservationSystem()
.getAllPlans().values(); .getAllPlans().values();

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
View File

@ -40,8 +40,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@ -161,13 +159,7 @@ public class TestApplicationCleanup {
Logger rootLogger = LogManager.getRootLogger(); Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG); rootLogger.setLevel(Level.DEBUG);
final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new MockRM();
MockRM rm = new MockRM() {
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
};
rm.start(); rm.start();
MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000);
@ -185,8 +177,8 @@ public class TestApplicationCleanup {
int request = 2; int request = 2;
am.allocate("127.0.0.1" , 1000, request, am.allocate("127.0.0.1" , 1000, request,
new ArrayList<ContainerId>()); new ArrayList<ContainerId>());
dispatcher.await(); rm.drainEvents();
//kick the scheduler //kick the scheduler
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(), List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(),
@ -199,7 +191,7 @@ public class TestApplicationCleanup {
Thread.sleep(100); Thread.sleep(100);
conts = am.allocate(new ArrayList<ResourceRequest>(), conts = am.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>()).getAllocatedContainers(); new ArrayList<ContainerId>()).getAllocatedContainers();
dispatcher.await(); rm.drainEvents();
contReceived += conts.size(); contReceived += conts.size();
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
} }
@ -209,7 +201,7 @@ public class TestApplicationCleanup {
ArrayList<ContainerId> release = new ArrayList<ContainerId>(); ArrayList<ContainerId> release = new ArrayList<ContainerId>();
release.add(conts.get(0).getId()); release.add(conts.get(0).getId());
am.allocate(new ArrayList<ResourceRequest>(), release); am.allocate(new ArrayList<ResourceRequest>(), release);
dispatcher.await(); rm.drainEvents();
// Send one more heartbeat with a fake running container. This is to // Send one more heartbeat with a fake running container. This is to
// simulate the situation that can happen if the NM reports that container // simulate the situation that can happen if the NM reports that container
@ -224,7 +216,7 @@ public class TestApplicationCleanup {
containerStatuses.put(app.getApplicationId(), containerStatusList); containerStatuses.put(app.getApplicationId(), containerStatusList);
NodeHeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true); NodeHeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true);
waitForContainerCleanup(dispatcher, nm1, resp); waitForContainerCleanup(rm, nm1, resp);
// Now to test the case when RM already gave cleanup, and NM suddenly // Now to test the case when RM already gave cleanup, and NM suddenly
// realizes that the container is running. // realizes that the container is running.
@ -240,17 +232,17 @@ public class TestApplicationCleanup {
resp = nm1.nodeHeartbeat(containerStatuses, true); resp = nm1.nodeHeartbeat(containerStatuses, true);
// The cleanup list won't be instantaneous as it is given out by scheduler // The cleanup list won't be instantaneous as it is given out by scheduler
// and not RMNodeImpl. // and not RMNodeImpl.
waitForContainerCleanup(dispatcher, nm1, resp); waitForContainerCleanup(rm, nm1, resp);
rm.stop(); rm.stop();
} }
protected void waitForContainerCleanup(DrainDispatcher dispatcher, MockNM nm, protected void waitForContainerCleanup(MockRM rm, MockNM nm,
NodeHeartbeatResponse resp) throws Exception { NodeHeartbeatResponse resp) throws Exception {
int waitCount = 0, cleanedConts = 0; int waitCount = 0, cleanedConts = 0;
List<ContainerId> contsToClean; List<ContainerId> contsToClean;
do { do {
dispatcher.await(); rm.drainEvents();
contsToClean = resp.getContainersToCleanup(); contsToClean = resp.getContainersToCleanup();
cleanedConts += contsToClean.size(); cleanedConts += contsToClean.size();
if (cleanedConts >= 1) { if (cleanedConts >= 1) {
@ -400,13 +392,7 @@ public class TestApplicationCleanup {
memStore.init(conf); memStore.init(conf);
// start RM // start RM
final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm1 = new MockRM(conf, memStore);
MockRM rm1 = new MockRM(conf, memStore) {
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
};
rm1.start(); rm1.start();
MockNM nm1 = MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
@ -419,13 +405,7 @@ public class TestApplicationCleanup {
rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING); rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING);
// start new RM // start new RM
final DrainDispatcher dispatcher2 = new DrainDispatcher(); MockRM rm2 = new MockRM(conf, memStore);
MockRM rm2 = new MockRM(conf, memStore) {
@Override
protected Dispatcher createDispatcher() {
return dispatcher2;
}
};
rm2.start(); rm2.start();
// nm1 register to rm2, and do a heartbeat // nm1 register to rm2, and do a heartbeat
@ -437,7 +417,7 @@ public class TestApplicationCleanup {
NodeHeartbeatResponse response = nm1.nodeHeartbeat(am0 NodeHeartbeatResponse response = nm1.nodeHeartbeat(am0
.getApplicationAttemptId(), 2, ContainerState.RUNNING); .getApplicationAttemptId(), 2, ContainerState.RUNNING);
waitForContainerCleanup(dispatcher2, nm1, response); waitForContainerCleanup(rm2, nm1, response);
rm1.stop(); rm1.stop();
rm2.stop(); rm2.stop();

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
View File

@ -59,8 +59,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException; import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
@ -261,7 +259,6 @@ public class TestApplicationMasterLauncher {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
conf.setInt(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 1); conf.setInt(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 1);
final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, null) { MockRM rm = new MockRMWithCustomAMLauncher(conf, null) {
@Override @Override
protected ApplicationMasterLauncher createAMLauncher() { protected ApplicationMasterLauncher createAMLauncher() {
@ -285,12 +282,8 @@ public class TestApplicationMasterLauncher {
} }
}; };
} }
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
}; };
rm.start(); rm.start();
MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120);
@ -298,7 +291,7 @@ public class TestApplicationMasterLauncher {
// kick the scheduling // kick the scheduling
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
MockRM.waitForState(app.getCurrentAppAttempt(), MockRM.waitForState(app.getCurrentAppAttempt(),
RMAppAttemptState.LAUNCHED, 500); RMAppAttemptState.LAUNCHED, 500);

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
View File

@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException; import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
@ -327,10 +325,8 @@ public class TestApplicationMasterService {
@Test(timeout=1200000) @Test(timeout=1200000)
public void testAllocateAfterUnregister() throws Exception { public void testAllocateAfterUnregister() throws Exception {
MyResourceManager rm = new MyResourceManager(conf); MockRM rm = new MockRM(conf);
rm.start(); rm.start();
DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext()
.getDispatcher();
// Register node1 // Register node1
MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
@ -351,7 +347,7 @@ public class TestApplicationMasterService {
AllocateResponse alloc1Response = am1.schedule(); AllocateResponse alloc1Response = am1.schedule();
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
rmDispatcher.await(); rm.drainEvents();
alloc1Response = am1.schedule(); alloc1Response = am1.schedule();
Assert.assertEquals(0, alloc1Response.getAllocatedContainers().size()); Assert.assertEquals(0, alloc1Response.getAllocatedContainers().size());
} }
@ -474,17 +470,6 @@ public class TestApplicationMasterService {
rm.stop(); rm.stop();
} }
private static class MyResourceManager extends MockRM {
public MyResourceManager(YarnConfiguration conf) {
super(conf);
}
@Override
protected Dispatcher createDispatcher() {
return new DrainDispatcher();
}
}
private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) { private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
RMContainer rmContainer = cs.getRMContainer(containerId); RMContainer rmContainer = cs.getRMContainer(containerId);

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
View File

@ -33,8 +33,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.TestAMRestart; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.TestAMRestart;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@ -65,8 +63,7 @@ public class TestNodeBlacklistingOnAMFailures {
conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
true); true);
DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = startRM(conf);
MockRM rm = startRM(conf, dispatcher);
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
// Register 5 nodes, so that we can blacklist atleast one if AM container // Register 5 nodes, so that we can blacklist atleast one if AM container
@ -122,7 +119,7 @@ public class TestNodeBlacklistingOnAMFailures {
// Try the current node a few times // Try the current node a few times
for (int i = 0; i <= 2; i++) { for (int i = 0; i <= 2; i++) {
currentNode.nodeHeartbeat(true); currentNode.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
Assert.assertEquals( Assert.assertEquals(
"AppAttemptState should still be SCHEDULED if currentNode is " "AppAttemptState should still be SCHEDULED if currentNode is "
@ -132,7 +129,7 @@ public class TestNodeBlacklistingOnAMFailures {
// Now try the other node // Now try the other node
otherNode.nodeHeartbeat(true); otherNode.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
// Now the AM container should be allocated // Now the AM container should be allocated
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
@ -169,8 +166,7 @@ public class TestNodeBlacklistingOnAMFailures {
conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
true); true);
DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = startRM(conf);
MockRM rm = startRM(conf, dispatcher);
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
// Register 5 nodes, so that we can blacklist atleast one if AM container // Register 5 nodes, so that we can blacklist atleast one if AM container
@ -227,7 +223,7 @@ public class TestNodeBlacklistingOnAMFailures {
System.out.println("New AppAttempt launched " + attempt.getAppAttemptId()); System.out.println("New AppAttempt launched " + attempt.getAppAttemptId());
nm2.nodeHeartbeat(true); nm2.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
// Now the AM container should be allocated // Now the AM container should be allocated
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
@ -257,8 +253,7 @@ public class TestNodeBlacklistingOnAMFailures {
conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED,
true); true);
DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = startRM(conf);
MockRM rm = startRM(conf, dispatcher);
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
// Register 5 nodes, so that we can blacklist atleast one if AM container // Register 5 nodes, so that we can blacklist atleast one if AM container
@ -319,7 +314,7 @@ public class TestNodeBlacklistingOnAMFailures {
nm3.nodeHeartbeat(true); nm3.nodeHeartbeat(true);
nm4.nodeHeartbeat(true); nm4.nodeHeartbeat(true);
nm5.nodeHeartbeat(true); nm5.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
// Now the AM container should be allocated // Now the AM container should be allocated
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
@ -352,8 +347,7 @@ public class TestNodeBlacklistingOnAMFailures {
1.5f); 1.5f);
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100);
DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = startRM(conf);
MockRM rm = startRM(conf, dispatcher);
MockNM node = MockNM node =
new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService()); new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService());
@ -367,7 +361,7 @@ public class TestNodeBlacklistingOnAMFailures {
// Now the AM container should be allocated // Now the AM container should be allocated
RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm); RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm);
node.nodeHeartbeat(true); node.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
rm.sendAMLaunched(attempt.getAppAttemptId()); rm.sendAMLaunched(attempt.getAppAttemptId());
rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
@ -394,7 +388,7 @@ public class TestNodeBlacklistingOnAMFailures {
.println("New AppAttempt launched " + attempt.getAppAttemptId()); .println("New AppAttempt launched " + attempt.getAppAttemptId());
node.nodeHeartbeat(true); node.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000);
rm.sendAMLaunched(attempt.getAppAttemptId()); rm.sendAMLaunched(attempt.getAppAttemptId());
@ -418,20 +412,13 @@ public class TestNodeBlacklistingOnAMFailures {
rm.waitForState(amAttemptID.getApplicationId(), RMAppState.ACCEPTED); rm.waitForState(amAttemptID.getApplicationId(), RMAppState.ACCEPTED);
} }
private MockRM startRM(YarnConfiguration conf, private MockRM startRM(YarnConfiguration conf) {
final DrainDispatcher dispatcher) {
MemoryRMStateStore memStore = new MemoryRMStateStore(); MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf); memStore.init(conf);
MockRM rm1 = new MockRM(conf, memStore) { MockRM rm = new MockRM(conf, memStore);
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
};
rm1.start(); rm.start();
return rm1; return rm;
} }
} }

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java
View File

@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@ -186,9 +185,7 @@ public class TestReservationSystemWithRMHA extends RMHATestBase {
rm.registerNode("127.0.0.1:1", memory, vCores); rm.registerNode("127.0.0.1:1", memory, vCores);
int attempts = 10; int attempts = 10;
do { do {
DrainDispatcher dispatcher = rm1.drainEvents();
(DrainDispatcher) rm1.getRMContext().getDispatcher();
dispatcher.await();
rm.getRMContext().getReservationSystem() rm.getRMContext().getReservationSystem()
.synchronizePlan(ReservationSystemTestUtil.reservationQ, false); .synchronizePlan(ReservationSystemTestUtil.reservationQ, false);
if (rm.getRMContext().getReservationSystem() if (rm.getRMContext().getReservationSystem()

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
View File

@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
@ -47,12 +45,10 @@ import org.junit.Test;
public class TestAMRMRPCNodeUpdates { public class TestAMRMRPCNodeUpdates {
private MockRM rm; private MockRM rm;
ApplicationMasterService amService = null; private ApplicationMasterService amService;
DrainDispatcher dispatcher = null;
@Before @Before
public void setUp() { public void setUp() {
dispatcher = new DrainDispatcher();
this.rm = new MockRM() { this.rm = new MockRM() {
@Override @Override
public void init(Configuration conf) { public void init(Configuration conf) {
@ -61,12 +57,8 @@ public class TestAMRMRPCNodeUpdates {
"1.0"); "1.0");
super.init(conf); super.init(conf);
} }
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
}; };
rm.start(); rm.start();
amService = rm.getApplicationMasterService(); amService = rm.getApplicationMasterService();
} }
@ -80,14 +72,14 @@ public class TestAMRMRPCNodeUpdates {
private void syncNodeHeartbeat(MockNM nm, boolean health) throws Exception { private void syncNodeHeartbeat(MockNM nm, boolean health) throws Exception {
nm.nodeHeartbeat(health); nm.nodeHeartbeat(health);
dispatcher.await(); rm.drainEvents();
} }
private void syncNodeLost(MockNM nm) throws Exception { private void syncNodeLost(MockNM nm) throws Exception {
rm.sendNodeStarted(nm); rm.sendNodeStarted(nm);
rm.waitForState(nm.getNodeId(), NodeState.RUNNING); rm.waitForState(nm.getNodeId(), NodeState.RUNNING);
rm.sendNodeLost(nm); rm.sendNodeLost(nm);
dispatcher.await(); rm.drainEvents();
} }
private AllocateResponse allocate(final ApplicationAttemptId attemptId, private AllocateResponse allocate(final ApplicationAttemptId attemptId,
@ -113,7 +105,7 @@ public class TestAMRMRPCNodeUpdates {
MockNM nm2 = rm.registerNode("127.0.0.2:1234", 10000); MockNM nm2 = rm.registerNode("127.0.0.2:1234", 10000);
MockNM nm3 = rm.registerNode("127.0.0.3:1234", 10000); MockNM nm3 = rm.registerNode("127.0.0.3:1234", 10000);
MockNM nm4 = rm.registerNode("127.0.0.4:1234", 10000); MockNM nm4 = rm.registerNode("127.0.0.4:1234", 10000);
dispatcher.await(); rm.drainEvents();
RMApp app1 = rm.submitApp(2000); RMApp app1 = rm.submitApp(2000);

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
View File

@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.event.InlineDispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
@ -228,21 +227,16 @@ public class TestNMReconnect extends ParameterizedSchedulerTestBase {
// The node(127.0.0.1:1234) reconnected with RM. When it registered with // The node(127.0.0.1:1234) reconnected with RM. When it registered with
// RM, RM set its lastNodeHeartbeatResponse's id to 0 asynchronously. But // RM, RM set its lastNodeHeartbeatResponse's id to 0 asynchronously. But
// the node's heartbeat come before RM succeeded setting the id to 0. // the node's heartbeat come before RM succeeded setting the id to 0.
final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new MockRM();
MockRM rm = new MockRM(){
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
};
rm.start(); rm.start();
MockNM nm1 = MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
nm1.registerNode(); nm1.registerNode();
int i = 0; int i = 0;
while(i < 3) { while(i < 3) {
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
i++; i++;
} }
@ -251,7 +245,7 @@ public class TestNMReconnect extends ParameterizedSchedulerTestBase {
nm2.registerNode(); nm2.registerNode();
RMNode rmNode = rm.getRMContext().getRMNodes().get(nm2.getNodeId()); RMNode rmNode = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
nm2.nodeHeartbeat(true); nm2.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
Assert.assertEquals("Node is Not in Running state.", NodeState.RUNNING, Assert.assertEquals("Node is Not in Running state.", NodeState.RUNNING,
rmNode.getState()); rmNode.getState());
rm.stop(); rm.stop();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
View File

@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockAM;

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
View File

@ -36,8 +36,6 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@ -612,24 +610,17 @@ public class TestApplicationPriority {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10); conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
final DrainDispatcher dispatcher = new DrainDispatcher();
MemoryRMStateStore memStore = new MemoryRMStateStore(); MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf); memStore.init(conf);
MockRM rm1 = new MockRM(conf, memStore) { MockRM rm1 = new MockRM(conf, memStore);
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
};
rm1.start(); rm1.start();
MockNM nm1 = MockNM nm1 =
new MockNM("127.0.0.1:1234", 16384, rm1.getResourceTrackerService()); new MockNM("127.0.0.1:1234", 16384, rm1.getResourceTrackerService());
nm1.registerNode(); nm1.registerNode();
rm1.drainEvents();
dispatcher.await();
ResourceScheduler scheduler = rm1.getRMContext().getScheduler(); ResourceScheduler scheduler = rm1.getRMContext().getScheduler();
LeafQueue defaultQueue = LeafQueue defaultQueue =
@ -648,7 +639,7 @@ public class TestApplicationPriority {
MockAM am2 = MockRM.launchAM(app2, rm1, nm1); MockAM am2 = MockRM.launchAM(app2, rm1, nm1);
am2.registerAppAttempt(); am2.registerAppAttempt();
dispatcher.await(); rm1.drainEvents();
Assert.assertEquals(2, defaultQueue.getNumActiveApplications()); Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
Assert.assertEquals(0, defaultQueue.getNumPendingApplications()); Assert.assertEquals(0, defaultQueue.getNumPendingApplications());
@ -657,7 +648,7 @@ public class TestApplicationPriority {
Priority appPriority3 = Priority.newInstance(7); Priority appPriority3 = Priority.newInstance(7);
RMApp app3 = rm1.submitApp(memory, appPriority3); RMApp app3 = rm1.submitApp(memory, appPriority3);
dispatcher.await(); rm1.drainEvents();
Assert.assertEquals(2, defaultQueue.getNumActiveApplications()); Assert.assertEquals(2, defaultQueue.getNumActiveApplications());
Assert.assertEquals(1, defaultQueue.getNumPendingApplications()); Assert.assertEquals(1, defaultQueue.getNumPendingApplications());
@ -676,14 +667,8 @@ public class TestApplicationPriority {
Assert.assertEquals(app3.getCurrentAppAttempt().getAppAttemptId(), Assert.assertEquals(app3.getCurrentAppAttempt().getAppAttemptId(),
fcApp3.getApplicationAttemptId()); fcApp3.getApplicationAttemptId());
final DrainDispatcher dispatcher1 = new DrainDispatcher();
// create new RM to represent restart and recover state // create new RM to represent restart and recover state
MockRM rm2 = new MockRM(conf, memStore) { MockRM rm2 = new MockRM(conf, memStore);
@Override
protected Dispatcher createDispatcher() {
return dispatcher1;
}
};
// start new RM // start new RM
rm2.start(); rm2.start();
@ -693,7 +678,7 @@ public class TestApplicationPriority {
// Verify RM Apps after this restart // Verify RM Apps after this restart
Assert.assertEquals(3, rm2.getRMContext().getRMApps().size()); Assert.assertEquals(3, rm2.getRMContext().getRMApps().size());
dispatcher1.await(); rm2.drainEvents();
scheduler = rm2.getRMContext().getScheduler(); scheduler = rm2.getRMContext().getScheduler();
defaultQueue = defaultQueue =
(LeafQueue) ((CapacityScheduler) scheduler).getQueue("default"); (LeafQueue) ((CapacityScheduler) scheduler).getQueue("default");
@ -714,7 +699,7 @@ public class TestApplicationPriority {
// NM resync to new RM // NM resync to new RM
nm1.registerNode(); nm1.registerNode();
dispatcher1.await(); rm2.drainEvents();
// wait for activating applications // wait for activating applications
count = 50; count = 50;

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
View File

@ -63,8 +63,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
@ -183,7 +181,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
StartContainersResponse mockResponse = mock(StartContainersResponse.class); StartContainersResponse mockResponse = mock(StartContainersResponse.class);
when(containerManager.startContainers((StartContainersRequest) any())) when(containerManager.startContainers((StartContainersRequest) any()))
.thenReturn(mockResponse); .thenReturn(mockResponse);
final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) { MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
protected ClientRMService createClientRMService() { protected ClientRMService createClientRMService() {
@ -192,11 +189,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
getRMContext().getRMDelegationTokenSecretManager()); getRMContext().getRMDelegationTokenSecretManager());
}; };
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
@Override @Override
protected void doSecureLogin() throws IOException { protected void doSecureLogin() throws IOException {
} }
@ -209,11 +201,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
// Set up a node. // Set up a node.
MockNM nm1 = rm.registerNode("localhost:1234", 3072); MockNM nm1 = rm.registerNode("localhost:1234", 3072);
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId(); ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
final MockAM mockAM = final MockAM mockAM =
@ -424,7 +415,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
StartContainersResponse mockResponse = mock(StartContainersResponse.class); StartContainersResponse mockResponse = mock(StartContainersResponse.class);
when(containerManager.startContainers((StartContainersRequest) any())) when(containerManager.startContainers((StartContainersRequest) any()))
.thenReturn(mockResponse); .thenReturn(mockResponse);
final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) { MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
protected ClientRMService createClientRMService() { protected ClientRMService createClientRMService() {
@ -433,11 +423,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
getRMContext().getRMDelegationTokenSecretManager()); getRMContext().getRMDelegationTokenSecretManager());
}; };
@Override
protected Dispatcher createDispatcher() {
return dispatcher;
}
@Override @Override
protected void doSecureLogin() throws IOException { protected void doSecureLogin() throws IOException {
} }
@ -450,10 +435,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
// Set up a node. // Set up a node.
MockNM nm1 = rm.registerNode("localhost:1234", 3072); MockNM nm1 = rm.registerNode("localhost:1234", 3072);
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
nm1.nodeHeartbeat(true); nm1.nodeHeartbeat(true);
dispatcher.await(); rm.drainEvents();
ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId(); ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
final MockAM mockAM = final MockAM mockAM =