From 6aaf33e50c065fa697e440c04df14a30967edb36 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Tue, 22 May 2018 13:25:15 -0700 Subject: [PATCH] YARN-8290. SystemMetricsPublisher.appACLsUpdated should be invoked after application information is published to ATS to avoid "User is not set in the application report" Exception. (Eric Yang via wangda) Change-Id: I0ac6ddd19740d1aa7dd07111cd11af71ddc2fcaf (cherry picked from commit bd15d2396ef0c24fb6b60c6393d16b37651b828e) --- .../hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 ----- .../hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java | 5 +++++ .../hadoop/yarn/server/resourcemanager/TestAppManager.java | 4 ---- .../hadoop/yarn/server/resourcemanager/TestRMRestart.java | 1 + 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 29830770ecf..3e64cfcfdf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -31,7 +31,6 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; @@ -466,10 +465,6 @@ public class RMAppManager implements EventHandler, // Inform the ACLs Manager this.applicationACLsManager.addApplication(applicationId, submissionContext.getAMContainerSpec().getApplicationACLs()); - String appViewACLs = submissionContext.getAMContainerSpec() - .getApplicationACLs().get(ApplicationAccessType.VIEW_APP); - rmContext.getSystemMetricsPublisher().appACLsUpdated( - application, appViewACLs, System.currentTimeMillis()); return application; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 4234e582573..c79e6fb5b23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -45,6 +45,7 @@ import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.StringInterner; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -1987,6 +1988,10 @@ public class RMAppImpl implements RMApp, Recoverable { private void sendATSCreateEvent() { rmContext.getRMApplicationHistoryWriter().applicationStarted(this); rmContext.getSystemMetricsPublisher().appCreated(this, this.startTime); + String appViewACLs = submissionContext.getAMContainerSpec() + .getApplicationACLs().get(ApplicationAccessType.VIEW_APP); + rmContext.getSystemMetricsPublisher().appACLsUpdated( + this, appViewACLs, System.currentTimeMillis()); } @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index f6cdfecd3cc..e79ba08ccd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -856,9 +856,6 @@ public class TestAppManager{ Assert.assertNotNull("app is null", app); Assert.assertEquals("app id doesn't match", appId, app.getApplicationId()); Assert.assertEquals("app state doesn't match", RMAppState.NEW, app.getState()); - verify(metricsPublisher).appACLsUpdated( - any(RMApp.class), any(String.class), anyLong()); - // wait for event to be processed int timeoutSecs = 0; while ((getAppEventType() == RMAppEventType.KILL) && @@ -867,7 +864,6 @@ public class TestAppManager{ } Assert.assertEquals("app event type sent is wrong", RMAppEventType.START, getAppEventType()); - return app; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index a4f18eea692..07c5268f533 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -766,6 +766,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { RMApp loadedApp0 = rm2.getRMContext().getRMApps().get(app0.getApplicationId()); rm2.waitForState(app0.getApplicationId(), RMAppState.FAILED); rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED); + Assert.assertEquals(app0.getUser(), loadedApp0.getUser()); // no new attempt is created. Assert.assertEquals(1, loadedApp0.getAppAttempts().size());