From cdab92d0c3912e9058835d54a013369bf65fadae Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Sat, 1 Jun 2013 00:15:53 +0000 Subject: [PATCH] merge YARN-735 from trunk. Make ApplicationAttemptId, ContainerId and NodeId immutable. Contributed by Jian He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1488440 13f79535-47bb-0310-9956-ffa450edef68 --- .../v2/app/local/LocalContainerAllocator.java | 12 +-- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 9 +-- .../mapreduce/v2/app/MRAppBenchmark.java | 13 ++-- .../hadoop/mapreduce/v2/app/MockJobs.java | 10 +-- .../v2/app/TestRuntimeEstimators.java | 5 +- .../mapreduce/v2/app/TestStagingCleanup.java | 26 ++----- .../v2/app/job/impl/TestJobImpl.java | 14 ++-- .../mapreduce/v2/app/webapp/TestAMWebApp.java | 2 +- .../v2/app/webapp/TestAMWebServices.java | 2 +- .../app/webapp/TestAMWebServicesAttempts.java | 2 +- .../app/webapp/TestAMWebServicesJobConf.java | 2 +- .../v2/app/webapp/TestAMWebServicesJobs.java | 2 +- .../v2/app/webapp/TestAMWebServicesTasks.java | 2 +- .../mapreduce/v2/hs/webapp/TestHSWebApp.java | 2 +- .../v2/hs/webapp/TestHsWebServices.java | 2 +- .../hs/webapp/TestHsWebServicesAttempts.java | 2 +- .../hs/webapp/TestHsWebServicesJobConf.java | 2 +- .../v2/hs/webapp/TestHsWebServicesJobs.java | 2 +- .../v2/hs/webapp/TestHsWebServicesTasks.java | 2 +- hadoop-yarn-project/CHANGES.txt | 3 + .../api/records/ApplicationAttemptId.java | 10 +-- .../hadoop/yarn/api/records/ContainerId.java | 10 +-- .../hadoop/yarn/api/records/NodeId.java | 9 +-- .../impl/pb/ApplicationAttemptIdPBImpl.java | 77 ++++++------------- .../records/impl/pb/ApplicationIdPBImpl.java | 3 + .../records/impl/pb/ContainerIdPBImpl.java | 73 ++++++------------ .../api/records/impl/pb/NodeIdPBImpl.java | 50 ++++++------ .../UnmanagedAMLauncher.java | 5 +- .../apache/hadoop/yarn/util/BuilderUtils.java | 30 +------- .../hadoop/yarn/util/ConverterUtils.java | 11 +-- .../java/org/apache/hadoop/yarn/MockApps.java | 9 --- .../yarn/server/nodemanager/NodeManager.java | 7 +- .../ContainerManagerImpl.java | 7 +- .../server/nodemanager/TestEventFlow.java | 10 +-- .../nodemanager/TestNodeManagerReboot.java | 9 +-- .../nodemanager/TestNodeManagerShutdown.java | 11 +-- .../nodemanager/TestNodeStatusUpdater.java | 21 ++--- .../BaseContainerManagerTest.java | 1 + .../TestContainerManager.java | 10 +-- .../launcher/TestContainerLaunch.java | 23 ++---- .../TestLogAggregationService.java | 9 +++ .../monitor/TestContainersMonitor.java | 9 +-- .../nodemanager/webapp/TestNMWebServices.java | 5 +- .../webapp/TestNMWebServicesApps.java | 5 +- .../resourcemanager/rmapp/RMAppImpl.java | 8 +- .../server/resourcemanager/Application.java | 7 +- .../server/resourcemanager/MockNodes.java | 11 +-- .../server/resourcemanager/NodeManager.java | 4 +- .../TestResourceTrackerService.java | 5 +- .../applicationsmanager/MockAsm.java | 6 +- .../resourcetracker/TestNMExpiry.java | 13 +--- .../TestRMNMRPCResponseId.java | 5 +- .../attempt/TestRMAppAttemptTransitions.java | 4 +- .../scheduler/fair/TestFSSchedulerApp.java | 9 +-- .../scheduler/fair/TestFairScheduler.java | 10 +-- .../scheduler/fifo/TestFifoScheduler.java | 6 +- 56 files changed, 219 insertions(+), 399 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java index 74ae16f0ff2..29ba7dac343 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java @@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.BuilderUtils; -import org.apache.hadoop.yarn.util.Records; /** * Allocates containers locally. Doesn't allocate a real container; @@ -134,16 +133,13 @@ public class LocalContainerAllocator extends RMCommunicator public void handle(ContainerAllocatorEvent event) { if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) { LOG.info("Processing the event " + event.toString()); - ContainerId cID = recordFactory.newRecordInstance(ContainerId.class); - cID.setApplicationAttemptId(applicationAttemptId); // Assign the same container ID as the AM - cID.setId(this.containerId.getId()); - + ContainerId cID = + ContainerId.newInstance(applicationAttemptId, + this.containerId.getId()); Container container = recordFactory.newRecordInstance(Container.class); container.setId(cID); - NodeId nodeId = Records.newRecord(NodeId.class); - nodeId.setHost(this.nmHost); - nodeId.setPort(this.nmPort); + NodeId nodeId = NodeId.newInstance(this.nmHost, this.nmPort); container.setNodeId(nodeId); container.setContainerToken(null); container.setNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index ee6e3b4957e..05ea74f3984 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -94,8 +94,6 @@ import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.state.StateMachine; @@ -122,9 +120,6 @@ public class MRApp extends MRAppMaster { public static int NM_PORT = 1234; public static int NM_HTTP_PORT = 8042; - private static final RecordFactory recordFactory = - RecordFactoryProvider.getRecordFactory(null); - //if true, tasks complete automatically as soon as they are launched protected boolean autoComplete = false; @@ -154,9 +149,7 @@ public class MRApp extends MRAppMaster { private static ApplicationAttemptId getApplicationAttemptId( ApplicationId applicationId, int startCount) { ApplicationAttemptId applicationAttemptId = - recordFactory.newRecordInstance(ApplicationAttemptId.class); - applicationAttemptId.setApplicationId(applicationId); - applicationAttemptId.setAttemptId(startCount); + ApplicationAttemptId.newInstance(applicationId, startCount); return applicationAttemptId; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index d17897ea93a..d52d8750391 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -139,19 +139,16 @@ public class MRAppBenchmark { try { if (concurrentRunningTasks < maxConcurrentRunningTasks) { event = eventQueue.take(); - ContainerId cId = - recordFactory.newRecordInstance(ContainerId.class); - cId.setApplicationAttemptId( - getContext().getApplicationAttemptId()); - cId.setId(containerCount++); + ContainerId cId = + ContainerId.newInstance(getContext() + .getApplicationAttemptId(), containerCount++); + //System.out.println("Allocating " + containerCount); Container container = recordFactory.newRecordInstance(Container.class); container.setId(cId); - NodeId nodeId = recordFactory.newRecordInstance(NodeId.class); - nodeId.setHost("dummy"); - nodeId.setPort(1234); + NodeId nodeId = NodeId.newInstance("dummy", 1234); container.setNodeId(nodeId); container.setContainerToken(null); container.setNodeHttpAddress("localhost:8042"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index c51ac65f87e..6014b0ff67e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -299,12 +299,10 @@ public class MockJobs extends MockApps { @Override public ContainerId getAssignedContainerID() { - ContainerId id = Records.newRecord(ContainerId.class); - ApplicationAttemptId appAttemptId = Records - .newRecord(ApplicationAttemptId.class); - appAttemptId.setApplicationId(taid.getTaskId().getJobId().getAppId()); - appAttemptId.setAttemptId(0); - id.setApplicationAttemptId(appAttemptId); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(taid.getTaskId().getJobId() + .getAppId(), 0); + ContainerId id = ContainerId.newInstance(appAttemptId, 0); return id; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index 35397284259..f7cdd4fedfc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -791,10 +791,7 @@ public class TestRuntimeEstimators { MyAppContext(int numberMaps, int numberReduces) { myApplicationID = ApplicationId.newInstance(clock.getTime(), 1); - myAppAttemptID = recordFactory - .newRecordInstance(ApplicationAttemptId.class); - myAppAttemptID.setApplicationId(myApplicationID); - myAppAttemptID.setAttemptId(0); + myAppAttemptID = ApplicationAttemptId.newInstance(myApplicationID, 0); myJobID = recordFactory.newRecordInstance(JobId.class); myJobID.setAppId(myApplicationID); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java index 3dca3e58791..8a0cf168e8b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java @@ -56,7 +56,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.service.AbstractService; -import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.Test; @@ -81,12 +80,9 @@ import org.junit.Test; String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); - ApplicationAttemptId attemptId = recordFactory.newRecordInstance( - ApplicationAttemptId.class); - attemptId.setAttemptId(0); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); - attemptId.setApplicationId(appId); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0); JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); @@ -109,12 +105,9 @@ import org.junit.Test; String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); - ApplicationAttemptId attemptId = recordFactory.newRecordInstance( - ApplicationAttemptId.class); - attemptId.setAttemptId(0); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); - attemptId.setApplicationId(appId); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, @@ -136,12 +129,9 @@ import org.junit.Test; String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); - ApplicationAttemptId attemptId = recordFactory.newRecordInstance( - ApplicationAttemptId.class); - attemptId.setAttemptId(1); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); - attemptId.setApplicationId(appId); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, JobStateInternal.REBOOT, 1); //no retry @@ -163,12 +153,9 @@ import org.junit.Test; String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); - ApplicationAttemptId attemptId = recordFactory.newRecordInstance( - ApplicationAttemptId.class); - attemptId.setAttemptId(0); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); - attemptId.setApplicationId(appId); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0); JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); @@ -190,12 +177,9 @@ import org.junit.Test; String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); - ApplicationAttemptId attemptId = recordFactory.newRecordInstance( - ApplicationAttemptId.class); - attemptId.setAttemptId(1); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); - attemptId.setApplicationId(appId); + ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java index 0b93e75546d..21fe3ad9b6e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java @@ -72,6 +72,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.SystemClock; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -557,10 +558,11 @@ public class TestJobImpl { JobID jobID = JobID.forName("job_1234567890000_0001"); JobId jobId = TypeConverter.toYarn(jobID); MRAppMetrics mrAppMetrics = MRAppMetrics.create(); - JobImpl job = new JobImpl(jobId, Records - .newRecord(ApplicationAttemptId.class), conf, mock(EventHandler.class), - null, new JobTokenSecretManager(), new Credentials(), null, null, - mrAppMetrics, null, true, null, 0, null, null, null, null); + JobImpl job = + new JobImpl(jobId, ApplicationAttemptId.newInstance( + ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class), + null, new JobTokenSecretManager(), new Credentials(), null, null, + mrAppMetrics, null, true, null, 0, null, null, null, null); InitTransition initTransition = getInitTransition(2); JobEvent mockJobEvent = mock(JobEvent.class); initTransition.transition(job, mockJobEvent); @@ -649,8 +651,8 @@ public class TestJobImpl { JobID jobID = JobID.forName("job_1234567890000_0001"); JobId jobId = TypeConverter.toYarn(jobID); StubbedJob job = new StubbedJob(jobId, - Records.newRecord(ApplicationAttemptId.class), conf, - dispatcher.getEventHandler(), true, "somebody", numSplits); + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0), + conf,dispatcher.getEventHandler(), true, "somebody", numSplits); dispatcher.register(JobEventType.class, job); EventHandler mockHandler = mock(EventHandler.class); dispatcher.register(TaskEventType.class, mockHandler); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java index 7622a9ab955..40e50bd709b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebApp.java @@ -55,7 +55,7 @@ public class TestAMWebApp { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java index 59a6fa630d4..5aa23524d09 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java @@ -83,7 +83,7 @@ public class TestAMWebServices extends JerseyTest { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java index d3d1419b32c..d3eb2eb6da9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java @@ -93,7 +93,7 @@ public class TestAMWebServicesAttempts extends JerseyTest { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java index 2e9abe770ac..00072a8e287 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java @@ -95,7 +95,7 @@ public class TestAMWebServicesJobConf extends JerseyTest { TestAppContext(int appid, int numTasks, int numAttempts, Path confPath) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); Map map = Maps.newHashMap(); Job job = MockJobs.newJob(appID, 0, numTasks, numAttempts, confPath); map.put(job.getID(), job); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java index 6917ed5122a..55d1a303127 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java @@ -96,7 +96,7 @@ public class TestAMWebServicesJobs extends JerseyTest { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java index ea3abda61b9..03a537e6d1b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java @@ -90,7 +90,7 @@ public class TestAMWebServicesTasks extends JerseyTest { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java index 8b2f1e833b3..787d8deb266 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java @@ -71,7 +71,7 @@ public class TestHSWebApp { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts, boolean hasFailedTasks) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts, hasFailedTasks); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java index 66b36632ca3..635974386e7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java @@ -89,7 +89,7 @@ public class TestHsWebServices extends JerseyTest { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java index a584987d427..621774b657e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java @@ -101,7 +101,7 @@ public class TestHsWebServicesAttempts extends JerseyTest { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java index d19a6468a0b..be08996b214 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java @@ -102,7 +102,7 @@ public class TestHsWebServicesJobConf extends JerseyTest { TestAppContext(int appid, int numTasks, int numAttempts, Path confPath) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); Map map = Maps.newHashMap(); Job job = MockJobs.newJob(appID, 0, numTasks, numAttempts, confPath); map.put(job.getID(), job); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java index bc5a41ab907..af9cfd21729 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java @@ -105,7 +105,7 @@ public class TestHsWebServicesJobs extends JerseyTest { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts, boolean hasFailedTasks) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); JobsPair jobs; try { jobs = MockHistoryJobs.newHistoryJobs(appID, numJobs, numTasks, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java index c8cdb7c22e3..f30dd648100 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java @@ -97,7 +97,7 @@ public class TestHsWebServicesTasks extends JerseyTest { TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); + appAttemptID = ApplicationAttemptId.newInstance(appID, 0); jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0b61325f3b1..8c5ed2b6e29 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -58,6 +58,9 @@ Release 2.0.5-beta - UNRELEASED ContainerTokenIdentifier instead of the entire Container. (Vinod Kumar Vavilapalli via sseth) + YARN-735. Make ApplicationAttemptId, ContaienrId and NodeId immutable. + (Jian He via sseth) + NEW FEATURES YARN-482. FS: Extend SchedulingMode to intermediate queues. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java index 0a33a5e909a..7b63e97afff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptId.java @@ -23,7 +23,6 @@ import java.text.NumberFormat; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; /** @@ -48,6 +47,7 @@ public abstract class ApplicationAttemptId implements Records.newRecord(ApplicationAttemptId.class); appAttemptId.setApplicationId(appId); appAttemptId.setAttemptId(attemptId); + appAttemptId.build(); return appAttemptId; } @@ -60,8 +60,7 @@ public abstract class ApplicationAttemptId implements public abstract ApplicationId getApplicationId(); @Private - @Unstable - public abstract void setApplicationId(ApplicationId appID); + protected abstract void setApplicationId(ApplicationId appID); /** * Get the attempt id of the Application. @@ -70,8 +69,7 @@ public abstract class ApplicationAttemptId implements public abstract int getAttemptId(); @Private - @Unstable - public abstract void setAttemptId(int attemptId); + protected abstract void setAttemptId(int attemptId); static final ThreadLocal attemptIdFormat = new ThreadLocal() { @@ -131,4 +129,6 @@ public abstract class ApplicationAttemptId implements sb.append("_").append(attemptIdFormat.get().format(getAttemptId())); return sb.toString(); } + + protected abstract void build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java index 99b36478887..95a989abd26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java @@ -23,7 +23,6 @@ import java.text.NumberFormat; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; /** @@ -40,6 +39,7 @@ public abstract class ContainerId implements Comparable{ ContainerId id = Records.newRecord(ContainerId.class); id.setId(containerId); id.setApplicationAttemptId(appAttemptId); + id.build(); return id; } @@ -54,8 +54,7 @@ public abstract class ContainerId implements Comparable{ public abstract ApplicationAttemptId getApplicationAttemptId(); @Private - @Unstable - public abstract void setApplicationAttemptId(ApplicationAttemptId atId); + protected abstract void setApplicationAttemptId(ApplicationAttemptId atId); /** * Get the identifier of the ContainerId. @@ -66,8 +65,7 @@ public abstract class ContainerId implements Comparable{ public abstract int getId(); @Private - @Unstable - public abstract void setId(int id); + protected abstract void setId(int id); // TODO: fail the app submission if attempts are more than 10 or something @@ -146,4 +144,6 @@ public abstract class ContainerId implements Comparable{ sb.append(containerIdFormat.get().format(getId())); return sb.toString(); } + + protected abstract void build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java index ae4ff89fa39..3c22c589b6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.api.records; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; /** @@ -40,6 +39,7 @@ public abstract class NodeId implements Comparable { NodeId nodeId = Records.newRecord(NodeId.class); nodeId.setHost(host); nodeId.setPort(port); + nodeId.build(); return nodeId; } @@ -52,8 +52,7 @@ public abstract class NodeId implements Comparable { public abstract String getHost(); @Private - @Unstable - public abstract void setHost(String host); + protected abstract void setHost(String host); /** * Get the port for communicating with the node. @@ -64,8 +63,7 @@ public abstract class NodeId implements Comparable { public abstract int getPort(); @Private - @Unstable - public abstract void setPort(int port); + protected abstract void setPort(int port); @Override public String toString() { @@ -111,4 +109,5 @@ public abstract class NodeId implements Comparable { return hostCompare; } + protected abstract void build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java index b11889910af..6cf3d03e213 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java @@ -21,15 +21,13 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import com.google.common.base.Preconditions; + public class ApplicationAttemptIdPBImpl extends ApplicationAttemptId { - ApplicationAttemptIdProto proto = ApplicationAttemptIdProto - .getDefaultInstance(); + ApplicationAttemptIdProto proto = null; ApplicationAttemptIdProto.Builder builder = null; - boolean viaProto = false; - private ApplicationId applicationId = null; public ApplicationAttemptIdPBImpl() { @@ -38,69 +36,36 @@ public class ApplicationAttemptIdPBImpl extends ApplicationAttemptId { public ApplicationAttemptIdPBImpl(ApplicationAttemptIdProto proto) { this.proto = proto; - viaProto = true; + this.applicationId = convertFromProtoFormat(proto.getApplicationId()); } - public synchronized ApplicationAttemptIdProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; + public ApplicationAttemptIdProto getProto() { return proto; } - private synchronized void mergeLocalToBuilder() { - if (this.applicationId != null - && !((ApplicationIdPBImpl) applicationId).getProto().equals( - builder.getApplicationId())) { - builder.setApplicationId(convertToProtoFormat(this.applicationId)); - } - } - - private synchronized void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private synchronized void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = ApplicationAttemptIdProto.newBuilder(proto); - } - viaProto = false; - } - - @Override - public synchronized int getAttemptId() { - ApplicationAttemptIdProtoOrBuilder p = viaProto ? proto : builder; - return (p.getAttemptId()); + public int getAttemptId() { + Preconditions.checkNotNull(proto); + return proto.getAttemptId(); } @Override - public synchronized void setAttemptId(int attemptId) { - maybeInitBuilder(); - builder.setAttemptId((attemptId)); + protected void setAttemptId(int attemptId) { + Preconditions.checkNotNull(builder); + builder.setAttemptId(attemptId); } + @Override - public synchronized ApplicationId getApplicationId() { - ApplicationAttemptIdProtoOrBuilder p = viaProto ? proto : builder; - if (this.applicationId != null) { - return this.applicationId; - } - if (!p.hasApplicationId()) { - return null; - } - this.applicationId = convertFromProtoFormat(p.getApplicationId()); + public ApplicationId getApplicationId() { return this.applicationId; } @Override - public synchronized void setApplicationId(ApplicationId appId) { - maybeInitBuilder(); - if (appId == null) - builder.clearApplicationId(); + public void setApplicationId(ApplicationId appId) { + if (appId != null) { + Preconditions.checkNotNull(builder); + builder.setApplicationId(convertToProtoFormat(appId)); + } this.applicationId = appId; } @@ -111,4 +76,10 @@ public class ApplicationAttemptIdPBImpl extends ApplicationAttemptId { private ApplicationIdProto convertToProtoFormat(ApplicationId t) { return ((ApplicationIdPBImpl)t).getProto(); } + + @Override + protected void build() { + proto = builder.build(); + builder = null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java index 031c194d6af..c758f54eea8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java @@ -49,6 +49,7 @@ public class ApplicationIdPBImpl extends ApplicationId { @Override protected void setId(int id) { + Preconditions.checkNotNull(builder); builder.setId(id); } @Override @@ -59,11 +60,13 @@ public class ApplicationIdPBImpl extends ApplicationId { @Override protected void setClusterTimestamp(long clusterTimestamp) { + Preconditions.checkNotNull(builder); builder.setClusterTimestamp((clusterTimestamp)); } @Override protected void build() { proto = builder.build(); + builder = null; } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java index 3227ce8a903..70492608824 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java @@ -22,14 +22,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProtoOrBuilder; + +import com.google.common.base.Preconditions; public class ContainerIdPBImpl extends ContainerId { - ContainerIdProto proto = ContainerIdProto.getDefaultInstance(); + ContainerIdProto proto = null; ContainerIdProto.Builder builder = null; - boolean viaProto = false; - private ApplicationAttemptId applicationAttemptId = null; public ContainerIdPBImpl() { @@ -38,71 +37,37 @@ public class ContainerIdPBImpl extends ContainerId { public ContainerIdPBImpl(ContainerIdProto proto) { this.proto = proto; - viaProto = true; + this.applicationAttemptId = convertFromProtoFormat(proto.getAppAttemptId()); } - public synchronized ContainerIdProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; + public ContainerIdProto getProto() { return proto; } - private synchronized void mergeLocalToBuilder() { - if (this.applicationAttemptId != null && ! - ((ApplicationAttemptIdPBImpl)applicationAttemptId).getProto().equals( - builder.getAppAttemptId())) { - builder.setAppAttemptId(convertToProtoFormat(this.applicationAttemptId)); - } - } - - private synchronized void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private synchronized void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = ContainerIdProto.newBuilder(proto); - } - viaProto = false; - } - - @Override - public synchronized int getId() { - ContainerIdProtoOrBuilder p = viaProto ? proto : builder; - return (p.getId()); + public int getId() { + Preconditions.checkNotNull(proto); + return proto.getId(); } @Override - public synchronized void setId(int id) { - maybeInitBuilder(); + protected void setId(int id) { + Preconditions.checkNotNull(builder); builder.setId((id)); } @Override - public synchronized ApplicationAttemptId getApplicationAttemptId() { - ContainerIdProtoOrBuilder p = viaProto ? proto : builder; - if (this.applicationAttemptId != null) { - return this.applicationAttemptId; - } - if (!p.hasAppAttemptId()) { - return null; - } - this.applicationAttemptId = convertFromProtoFormat(p.getAppAttemptId()); + public ApplicationAttemptId getApplicationAttemptId() { return this.applicationAttemptId; } @Override - public synchronized void setApplicationAttemptId(ApplicationAttemptId atId) { - maybeInitBuilder(); - if (atId == null) - builder.clearAppAttemptId(); + protected void setApplicationAttemptId(ApplicationAttemptId atId) { + if (atId != null) { + Preconditions.checkNotNull(builder); + builder.setAppAttemptId(convertToProtoFormat(atId)); + } this.applicationAttemptId = atId; } @@ -115,4 +80,10 @@ public class ContainerIdPBImpl extends ContainerId { ApplicationAttemptId t) { return ((ApplicationAttemptIdPBImpl)t).getProto(); } + + @Override + protected void build() { + proto = builder.build(); + builder = null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java index f0a7830117a..d406b103f94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java @@ -21,14 +21,14 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProtoOrBuilder; + +import com.google.common.base.Preconditions; public class NodeIdPBImpl extends NodeId { - NodeIdProto proto = NodeIdProto.getDefaultInstance(); + NodeIdProto proto = null; NodeIdProto.Builder builder = null; - boolean viaProto = false; public NodeIdPBImpl() { builder = NodeIdProto.newBuilder(); @@ -36,43 +36,39 @@ public class NodeIdPBImpl extends NodeId { public NodeIdPBImpl(NodeIdProto proto) { this.proto = proto; - viaProto = true; } - - public synchronized NodeIdProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; + + public NodeIdProto getProto() { return proto; } - private synchronized void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = NodeIdProto.newBuilder(proto); - } - viaProto = false; - } - @Override - public synchronized String getHost() { - NodeIdProtoOrBuilder p = viaProto ? proto : builder; - return (p.getHost()); + public String getHost() { + Preconditions.checkNotNull(proto); + return proto.getHost(); } @Override - public synchronized void setHost(String host) { - maybeInitBuilder(); - builder.setHost((host)); + protected void setHost(String host) { + Preconditions.checkNotNull(builder); + builder.setHost(host); } @Override - public synchronized int getPort() { - NodeIdProtoOrBuilder p = viaProto ? proto : builder; - return (p.getPort()); + public int getPort() { + Preconditions.checkNotNull(proto); + return proto.getPort(); } @Override - public synchronized void setPort(int port) { - maybeInitBuilder(); - builder.setPort((port)); + protected void setPort(int port) { + Preconditions.checkNotNull(builder); + builder.setPort(port); + } + + @Override + protected void build() { + proto = builder.build(); + builder = null; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index 5805e2bead9..a5de27a51f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -185,10 +185,7 @@ public class UnmanagedAMLauncher { if(!setClasspath && classpath!=null) { envAMList.add("CLASSPATH="+classpath); } - - ContainerId containerId = Records.newRecord(ContainerId.class); - containerId.setApplicationAttemptId(attemptId); - containerId.setId(0); + ContainerId containerId = ContainerId.newInstance(attemptId, 0); String hostname = InetAddress.getLocalHost().getHostName(); envAMList.add(Environment.CONTAINER_ID.name() + "=" + containerId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index 1e65767709e..943b7c1e0ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -147,11 +147,7 @@ public class BuilderUtils { public static ApplicationAttemptId newApplicationAttemptId( ApplicationId appId, int attemptId) { - ApplicationAttemptId appAttemptId = - recordFactory.newRecordInstance(ApplicationAttemptId.class); - appAttemptId.setApplicationId(appId); - appAttemptId.setAttemptId(attemptId); - return appAttemptId; + return ApplicationAttemptId.newInstance(appId, attemptId); } public static ApplicationId convert(long clustertimestamp, CharSequence id) { @@ -161,10 +157,7 @@ public class BuilderUtils { public static ContainerId newContainerId(ApplicationAttemptId appAttemptId, int containerId) { - ContainerId id = recordFactory.newRecordInstance(ContainerId.class); - id.setId(containerId); - id.setApplicationAttemptId(appAttemptId); - return id; + return ContainerId.newInstance(appAttemptId, containerId); } public static ContainerId newContainerId(int appId, int appAttemptId, @@ -189,26 +182,11 @@ public class BuilderUtils { public static ContainerId newContainerId(RecordFactory recordFactory, ApplicationId appId, ApplicationAttemptId appAttemptId, int containerId) { - ContainerId id = recordFactory.newRecordInstance(ContainerId.class); - id.setId(containerId); - id.setApplicationAttemptId(appAttemptId); - return id; - } - - public static ContainerId newContainerId(RecordFactory recordFactory, - ApplicationAttemptId appAttemptId, - int containerId) { - ContainerId id = recordFactory.newRecordInstance(ContainerId.class); - id.setApplicationAttemptId(appAttemptId); - id.setId(containerId); - return id; + return ContainerId.newInstance(appAttemptId, containerId); } public static NodeId newNodeId(String host, int port) { - NodeId nodeId = recordFactory.newRecordInstance(NodeId.class); - nodeId.setHost(host); - nodeId.setPort(port); - return nodeId; + return NodeId.newInstance(host, port); } public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index 9a367087dcf..b59f1dce111 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -123,10 +123,8 @@ public class ConverterUtils { Iterator it) throws NumberFormatException { ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), Integer.parseInt(it.next())); - ApplicationAttemptId appAttemptId = Records - .newRecord(ApplicationAttemptId.class); - appAttemptId.setApplicationId(appId); - appAttemptId.setAttemptId(Integer.parseInt(it.next())); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next())); return appAttemptId; } @@ -164,9 +162,8 @@ public class ConverterUtils { } try { ApplicationAttemptId appAttemptID = toApplicationAttemptId(it); - ContainerId containerId = Records.newRecord(ContainerId.class); - containerId.setApplicationAttemptId(appAttemptID); - containerId.setId(Integer.parseInt(it.next())); + ContainerId containerId = + ContainerId.newInstance(appAttemptID, Integer.parseInt(it.next())); return containerId; } catch (NumberFormatException n) { throw new IllegalArgumentException("Invalid ContainerId: " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java index ecfac4d3f3c..401cfec2e99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java @@ -20,10 +20,8 @@ package org.apache.hadoop.yarn; import java.util.Iterator; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.util.Records; import com.google.common.collect.Iterators; @@ -65,13 +63,6 @@ public class MockApps { return ApplicationId.newInstance(TS, i); } - public static ApplicationAttemptId newAppAttemptID(ApplicationId appId, int i) { - ApplicationAttemptId id = Records.newRecord(ApplicationAttemptId.class); - id.setApplicationId(appId); - id.setAttemptId(i); - return id; - } - public static YarnApplicationState newAppState() { synchronized(STATES) { return STATES.next(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 2ae578b1dda..e310b9450c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -55,7 +55,6 @@ import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecret import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.service.CompositeService; -import org.apache.hadoop.yarn.util.Records; import com.google.common.annotations.VisibleForTesting; @@ -288,7 +287,7 @@ public class NodeManager extends CompositeService public static class NMContext implements Context { - private final NodeId nodeId = Records.newRecord(NodeId.class); + private NodeId nodeId = null; private final ConcurrentMap applications = new ConcurrentHashMap(); private final ConcurrentMap containers = @@ -351,6 +350,10 @@ public class NodeManager extends CompositeService public void setWebServer(WebServer webServer) { this.webServer = webServer; } + + public void setNodeId(NodeId nodeId) { + this.nodeId = nodeId; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 89ccb871c76..6a7bd683f3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.ContainerToken; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -71,6 +72,7 @@ import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent; @@ -249,8 +251,9 @@ public class ContainerManagerImpl extends CompositeService implements this.setBlockNewContainerRequests(true); server.start(); InetSocketAddress connectAddress = NetUtils.getConnectAddress(server); - this.context.getNodeId().setHost(connectAddress.getHostName()); - this.context.getNodeId().setPort(connectAddress.getPort()); + NodeId nodeId = NodeId.newInstance(connectAddress.getHostName(), + connectAddress.getPort()); + ((NodeManager.NMContext)context).setNodeId(nodeId); LOG.info("ContainerManager started at " + connectAddress); super.start(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 06830d7eb96..b4695960a99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -126,13 +126,11 @@ public class TestEventFlow { ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - ContainerId cID = recordFactory.newRecordInstance(ContainerId.class); ApplicationId applicationId = ApplicationId.newInstance(0, 0); - ApplicationAttemptId applicationAttemptId = - recordFactory.newRecordInstance(ApplicationAttemptId.class); - applicationAttemptId.setApplicationId(applicationId); - applicationAttemptId.setAttemptId(0); - cID.setApplicationAttemptId(applicationAttemptId); + ApplicationAttemptId applicationAttemptId = + ApplicationAttemptId.newInstance(applicationId, 0); + ContainerId cID = ContainerId.newInstance(applicationAttemptId, 0); + Resource r = BuilderUtils.newResource(1024, 1); String user = "testing"; String host = "127.0.0.1"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index d97c664bdc1..f9f7e2339d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -243,13 +243,8 @@ public class TestNodeManagerReboot { private ContainerId createContainerId() { ApplicationId appId = ApplicationId.newInstance(0, 0); - ApplicationAttemptId appAttemptId = - Records.newRecord(ApplicationAttemptId.class); - appAttemptId.setApplicationId(appId); - appAttemptId.setAttemptId(1); - ContainerId containerId = - Records.newRecord(ContainerId.class); - containerId.setApplicationAttemptId(appAttemptId); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newInstance(appAttemptId, 0); return containerId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index defb5fa01be..ca87d624eae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -212,14 +212,9 @@ public class TestNodeManagerShutdown { public static ContainerId createContainerId() { ApplicationId appId = ApplicationId.newInstance(0, 0); - - ApplicationAttemptId appAttemptId = - recordFactory.newRecordInstance(ApplicationAttemptId.class); - appAttemptId.setApplicationId(appId); - appAttemptId.setAttemptId(1); - ContainerId containerId = - recordFactory.newRecordInstance(ContainerId.class); - containerId.setApplicationAttemptId(appAttemptId); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newInstance(appAttemptId, 0); return containerId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 777d92f4bb7..4e150c475cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -152,13 +152,6 @@ public class TestNodeStatusUpdater { return response; } - ApplicationAttemptId appAttemptID = recordFactory - .newRecordInstance(ApplicationAttemptId.class); - ContainerId firstContainerID = recordFactory - .newRecordInstance(ContainerId.class); - ContainerId secondContainerID = recordFactory - .newRecordInstance(ContainerId.class); - private Map> getAppToContainerStatusMap( List containers) { Map> map = @@ -196,9 +189,10 @@ public class TestNodeStatusUpdater { Assert.assertEquals(0, nodeStatus.getContainersStatuses().size()); // Give a container to the NM. - appAttemptID.setApplicationId(appId1); - firstContainerID.setApplicationAttemptId(appAttemptID); - firstContainerID.setId(heartBeatID); + ApplicationAttemptId appAttemptID = + ApplicationAttemptId.newInstance(appId1, 0); + ContainerId firstContainerID = + ContainerId.newInstance(appAttemptID, heartBeatID); ContainerLaunchContext launchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); Resource resource = BuilderUtils.newResource(2, 1); @@ -226,9 +220,10 @@ public class TestNodeStatusUpdater { Assert.assertEquals(1, activeContainers.size()); // Give another container to the NM. - appAttemptID.setApplicationId(appId2); - secondContainerID.setApplicationAttemptId(appAttemptID); - secondContainerID.setId(heartBeatID); + ApplicationAttemptId appAttemptID = + ApplicationAttemptId.newInstance(appId2, 0); + ContainerId secondContainerID = + ContainerId.newInstance(appAttemptID, heartBeatID); ContainerLaunchContext launchContext = recordFactory .newRecordInstance(ContainerLaunchContext.class); long currentTime = System.currentTimeMillis(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index 1dd8cc27fc9..eae48ab09ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index 90bd4705774..6b3e27baeaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -75,13 +75,9 @@ public class TestContainerManager extends BaseContainerManagerTest { private ContainerId createContainerId() { ApplicationId appId = ApplicationId.newInstance(0, 0); - ApplicationAttemptId appAttemptId = - recordFactory.newRecordInstance(ApplicationAttemptId.class); - appAttemptId.setApplicationId(appId); - appAttemptId.setAttemptId(1); - ContainerId containerId = - recordFactory.newRecordInstance(ContainerId.class); - containerId.setApplicationAttemptId(appAttemptId); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newInstance(appAttemptId, 0); return containerId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index ff4e4937307..1941e4e6f30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -164,15 +164,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest { // ////// Construct the Container-id ApplicationId appId = ApplicationId.newInstance(0, 0); - ApplicationAttemptId appAttemptId = - recordFactory.newRecordInstance(ApplicationAttemptId.class); - appAttemptId.setApplicationId(appId); - appAttemptId.setAttemptId(1); - ContainerId cId = - recordFactory.newRecordInstance(ContainerId.class); - int port = 12345; - cId.setApplicationAttemptId(appAttemptId); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + int port = 12345; + ContainerId cId = ContainerId.newInstance(appAttemptId, 0); Map userSetEnv = new HashMap(); userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id"); userSetEnv.put(Environment.NM_HOST.name(), "user_set_NM_HOST"); @@ -321,14 +317,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest { // ////// Construct the Container-id ApplicationId appId = ApplicationId.newInstance(1, 1); - ApplicationAttemptId appAttemptId = - recordFactory.newRecordInstance(ApplicationAttemptId.class); - appAttemptId.setApplicationId(appId); - appAttemptId.setAttemptId(1); - ContainerId cId = - recordFactory.newRecordInstance(ContainerId.class); - cId.setApplicationAttemptId(appAttemptId); - + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId cId = ContainerId.newInstance(appAttemptId, 0); File processStartFile = new File(tmpDir, "pid.txt").getAbsoluteFile(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 05bc408fee2..49be907aa7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -78,6 +79,7 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; @@ -113,6 +115,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest { this.remoteRootLogDir.mkdir(); } + @Override + public void setup() throws IOException { + super.setup(); + NodeId nodeId = NodeId.newInstance("0.0.0.0", 5555); + ((NMContext)context).setNodeId(nodeId); + } + @Override public void tearDown() throws IOException, InterruptedException { super.tearDown(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 27539d378ae..13fbf98855e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -202,13 +202,8 @@ public class TestContainersMonitor extends BaseContainerManagerTest { recordFactory.newRecordInstance(ContainerLaunchContext.class); // ////// Construct the Container-id ApplicationId appId = ApplicationId.newInstance(0, 0); - ApplicationAttemptId appAttemptId = - recordFactory.newRecordInstance(ApplicationAttemptId.class); - appAttemptId.setApplicationId(appId); - appAttemptId.setAttemptId(1); - ContainerId cId = recordFactory.newRecordInstance(ContainerId.class); - cId.setId(0); - cId.setApplicationAttemptId(appAttemptId); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); + ContainerId cId = ContainerId.newInstance(appAttemptId, 0); int port = 12345; URL resource_alpha = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index 08768337ed4..bddb9d10c1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -32,6 +32,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; @@ -86,8 +87,8 @@ public class TestNMWebServices extends JerseyTest { @Override protected void configureServlets() { nmContext = new NodeManager.NMContext(null); - nmContext.getNodeId().setHost("testhost.foo.com"); - nmContext.getNodeId().setPort(8042); + NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042); + ((NodeManager.NMContext)nmContext).setNodeId(nodeId); resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index 9b059bb9bba..d386c44ccc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -34,6 +34,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; @@ -93,8 +94,8 @@ public class TestNMWebServicesApps extends JerseyTest { @Override protected void configureServlets() { nmContext = new NodeManager.NMContext(null); - nmContext.getNodeId().setHost("testhost.foo.com"); - nmContext.getNodeId().setPort(9999); + NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999); + ((NodeManager.NMContext)nmContext).setNodeId(nodeId); resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index db68c29d121..b5c5215bc94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -71,7 +71,6 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.BuilderUtils; -import org.apache.hadoop.yarn.util.Records; public class RMAppImpl implements RMApp, Recoverable { @@ -579,11 +578,8 @@ public class RMAppImpl implements RMApp, Recoverable { @SuppressWarnings("unchecked") private void createNewAttempt(boolean startAttempt) { - ApplicationAttemptId appAttemptId = Records - .newRecord(ApplicationAttemptId.class); - appAttemptId.setApplicationId(applicationId); - appAttemptId.setAttemptId(attempts.size() + 1); - + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(applicationId, attempts.size() + 1); RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService, submissionContext, conf, user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 2c70c0efcfe..8e49144262e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.Task.State; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; -import org.apache.hadoop.yarn.util.Records; @Private public class Application { @@ -99,9 +98,9 @@ public class Application { this.resourceManager = resourceManager; this.applicationId = this.resourceManager.getClientRMService().getNewApplicationId(); - this.applicationAttemptId = Records.newRecord(ApplicationAttemptId.class); - this.applicationAttemptId.setApplicationId(this.applicationId); - this.applicationAttemptId.setAttemptId(this.numAttempts.getAndIncrement()); + this.applicationAttemptId = + ApplicationAttemptId.newInstance(this.applicationId, + this.numAttempts.getAndIncrement()); } public String getUser() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index d177edbffde..53402b4e73a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.hadoop.net.Node; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -71,13 +70,6 @@ public class MockNodes { return list; } - public static NodeId newNodeID(String host, int port) { - NodeId nid = recordFactory.newRecordInstance(NodeId.class); - nid.setHost(host); - nid.setPort(port); - return nid; - } - public static Resource newResource(int mem) { Resource rs = recordFactory.newRecordInstance(Resource.class); rs.setMemory(mem); @@ -214,7 +206,8 @@ public class MockNodes { if (hostName == null) { hostName = "host"+ nid; } - final NodeId nodeID = newNodeID(hostName, port); + final NodeId nodeID = NodeId.newInstance(hostName, port); + final String httpAddress = httpAddr; final NodeHealthStatus nodeHealthStatus = recordFactory.newRecordInstance(NodeHealthStatus.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index 24261c29b55..a7035a8a3dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -91,9 +91,7 @@ public class NodeManager implements ContainerManager { this.capability = capability; Resources.addTo(available, capability); - this.nodeId = recordFactory.newRecordInstance(NodeId.class); - this.nodeId.setHost(hostName); - this.nodeId.setPort(containerManagerPort); + this.nodeId = NodeId.newInstance(hostName, containerManagerPort); RegisterNodeManagerRequest request = recordFactory .newRecordInstance(RegisterNodeManagerRequest.class); request.setHttpPort(httpPort); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 5f12b337715..bc8669999c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; @@ -260,9 +259,7 @@ public class TestResourceTrackerService { ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService(); RegisterNodeManagerRequest req = Records.newRecord( RegisterNodeManagerRequest.class); - NodeId nodeId = Records.newRecord(NodeId.class); - nodeId.setHost("host2"); - nodeId.setPort(1234); + NodeId nodeId = NodeId.newInstance("host2", 1234); req.setNodeId(nodeId); req.setHttpPort(1234); // trying to register a invalid node. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 3c99b449550..fcddf63cff2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -139,10 +139,10 @@ public abstract class MockAsm extends MockApps { } public static RMApp newApplication(int i) { - final ApplicationAttemptId appAttemptId = newAppAttemptID(newAppID(i), 0); + final ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(newAppID(i), 0); final Container masterContainer = Records.newRecord(Container.class); - ContainerId containerId = Records.newRecord(ContainerId.class); - containerId.setApplicationAttemptId(appAttemptId); + ContainerId containerId = ContainerId.newInstance(appAttemptId, 0); masterContainer.setId(containerId); masterContainer.setNodeHttpAddress("node:port"); final String user = newUserName(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index 9f71c7ed905..595d5d8f894 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; -import org.apache.hadoop.yarn.util.Records; import org.junit.Before; import org.junit.Test; @@ -132,9 +131,7 @@ public class TestNMExpiry { RegisterNodeManagerRequest request1 = recordFactory .newRecordInstance(RegisterNodeManagerRequest.class); - NodeId nodeId1 = Records.newRecord(NodeId.class); - nodeId1.setPort(0); - nodeId1.setHost(hostname1); + NodeId nodeId1 = NodeId.newInstance(hostname1, 0); request1.setNodeId(nodeId1); request1.setHttpPort(0); request1.setResource(capability); @@ -142,9 +139,7 @@ public class TestNMExpiry { RegisterNodeManagerRequest request2 = recordFactory .newRecordInstance(RegisterNodeManagerRequest.class); - NodeId nodeId2 = Records.newRecord(NodeId.class); - nodeId2.setPort(0); - nodeId2.setHost(hostname2); + NodeId nodeId2 = NodeId.newInstance(hostname2, 0); request2.setNodeId(nodeId2); request2.setHttpPort(0); request2.setResource(capability); @@ -160,9 +155,7 @@ public class TestNMExpiry { request3 = recordFactory .newRecordInstance(RegisterNodeManagerRequest.class); - NodeId nodeId3 = Records.newRecord(NodeId.class); - nodeId3.setPort(0); - nodeId3.setHost(hostname3); + NodeId nodeId3 = NodeId.newInstance(hostname3, 0); request3.setNodeId(nodeId3); request3.setHttpPort(0); request3.setResource(capability); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index e2ffa786cc0..04c24fcf680 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; -import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -93,9 +92,7 @@ public class TestRMNMRPCResponseId { String node = "localhost"; Resource capability = BuilderUtils.newResource(1024, 1); RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class); - nodeId = Records.newRecord(NodeId.class); - nodeId.setHost(node); - nodeId.setPort(1234); + nodeId = NodeId.newInstance(node, 1234); request.setNodeId(nodeId); request.setHttpPort(0); request.setResource(capability); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 3f82db1a405..8de3397ac08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -194,8 +194,8 @@ public class TestRMAppAttemptTransitions { ApplicationId applicationId = MockApps.newAppID(appId++); - ApplicationAttemptId applicationAttemptId = - MockApps.newAppAttemptID(applicationId, 0); + ApplicationAttemptId applicationAttemptId = + ApplicationAttemptId.newInstance(applicationId, 0); final String user = MockApps.newUserName(); final String queue = MockApps.newQueue(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java index 3b545193a53..8a53bd036be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java @@ -23,22 +23,17 @@ import static org.junit.Assert.assertEquals; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerApp; import org.junit.Test; import org.mockito.Mockito; public class TestFSSchedulerApp { - private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { - ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class); ApplicationId appIdImpl = ApplicationId.newInstance(0, appId); - attId.setAttemptId(attemptId); - attId.setApplicationId(appIdImpl); + ApplicationAttemptId attId = + ApplicationAttemptId.newInstance(appIdImpl, attemptId); return attId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 65a408a7062..acc3c57b34c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -142,10 +142,9 @@ public class TestFairScheduler { } private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { - ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class); ApplicationId appIdImpl = ApplicationId.newInstance(0, appId); - attId.setAttemptId(attemptId); - attId.setApplicationId(appIdImpl); + ApplicationAttemptId attId = + ApplicationAttemptId.newInstance(appIdImpl, attemptId); return attId; } @@ -1642,9 +1641,8 @@ public class TestFairScheduler { assertEquals("The application doesn't reach SUBMITTED.", RMAppState.SUBMITTED, application.getState()); - ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class); - attId.setAttemptId(this.ATTEMPT_ID++); - attId.setApplicationId(applicationId); + ApplicationAttemptId attId = + ApplicationAttemptId.newInstance(applicationId, this.ATTEMPT_ID++); scheduler.addApplication(attId, queue, user); numTries = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 0420780aded..886c0733719 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -95,11 +95,9 @@ public class TestFifoScheduler { } private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { - ApplicationAttemptId attId = recordFactory - .newRecordInstance(ApplicationAttemptId.class); ApplicationId appIdImpl = ApplicationId.newInstance(0, appId); - attId.setAttemptId(attemptId); - attId.setApplicationId(appIdImpl); + ApplicationAttemptId attId = + ApplicationAttemptId.newInstance(appIdImpl, attemptId); return attId; }