From 81c9d17af84ed87b9ded7057cb726a3855ddd32d Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Mon, 17 Nov 2014 21:13:22 +0000 Subject: [PATCH] YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created. Contributed by Wangda Tan --- hadoop-yarn-project/CHANGES.txt | 3 + .../security/ApplicationACLsManager.java | 7 ++ .../security/QueueACLsManager.java | 7 ++ .../resourcemanager/webapp/AppBlock.java | 30 +++++-- .../resourcemanager/webapp/TestAppPage.java | 90 +++++++++++++++++++ 5 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b303e53607f..0f526ddde09 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -99,6 +99,9 @@ Release 2.7.0 - UNRELEASED YARN-1703. Fixed ResourceManager web-proxy to close connections correctly. (Rohith Sharma via vinodkv) + YARN-2414. RM web UI: app page will crash if app is failed before any + attempt has been created (Wangda Tan via jlowe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java index e8e3cb56b66..4cacfca8e2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java @@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AdminACLsManager; +import com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.Private public class ApplicationACLsManager { @@ -48,6 +50,11 @@ public class ApplicationACLsManager { private final ConcurrentMap> applicationACLS = new ConcurrentHashMap>(); + @VisibleForTesting + public ApplicationACLsManager() { + this(new Configuration()); + } + public ApplicationACLsManager(Configuration conf) { this.conf = conf; this.adminAclsManager = new AdminACLsManager(this.conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java index eb5037afa1b..fb8279d7d14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java @@ -24,9 +24,16 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import com.google.common.annotations.VisibleForTesting; + public class QueueACLsManager { private ResourceScheduler scheduler; private boolean isACLsEnable; + + @VisibleForTesting + public QueueACLsManager() { + this(null, new Configuration()); + } public QueueACLsManager(ResourceScheduler scheduler, Configuration conf) { this.scheduler = scheduler; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index c427ccfc94d..a108e435042 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -33,6 +33,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; @@ -113,8 +115,23 @@ public class AppBlock extends HtmlBlock { setTitle(join("Application ", aid)); RMAppMetrics appMerics = rmApp.getRMAppMetrics(); - RMAppAttemptMetrics attemptMetrics = - rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics(); + + // Get attempt metrics and fields, it is possible currentAttempt of RMApp is + // null. In that case, we will assume resource preempted and number of Non + // AM container preempted on that attempt is 0 + RMAppAttemptMetrics attemptMetrics; + if (null == rmApp.getCurrentAppAttempt()) { + attemptMetrics = null; + } else { + attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics(); + } + Resource attemptResourcePreempted = + attemptMetrics == null ? Resources.none() : attemptMetrics + .getResourcePreempted(); + int attemptNumNonAMContainerPreempted = + attemptMetrics == null ? 0 : attemptMetrics + .getNumNonAMContainersPreempted(); + info("Application Overview") ._("User:", app.getUser()) ._("Name:", app.getName()) @@ -143,13 +160,12 @@ public class AppBlock extends HtmlBlock { ._("Total Number of AM Containers Preempted:", String.valueOf(appMerics.getNumAMContainersPreempted())) ._("Resource Preempted from Current Attempt:", - attemptMetrics.getResourcePreempted()) + attemptResourcePreempted) ._("Number of Non-AM Containers Preempted from Current Attempt:", - String.valueOf(attemptMetrics - .getNumNonAMContainersPreempted())) + attemptNumNonAMContainerPreempted) ._("Aggregate Resource Allocation:", - String.format("%d MB-seconds, %d vcore-seconds", - appMerics.getMemorySeconds(), appMerics.getVcoreSeconds())); + String.format("%d MB-seconds, %d vcore-seconds", + appMerics.getMemorySeconds(), appMerics.getVcoreSeconds())); pdiv._(); Collection attempts = rmApp.getAppAttempts().values(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java new file mode 100644 index 00000000000..9732c193b51 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.test.WebAppTests; +import org.junit.Test; + +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Module; + +public class TestAppPage { + @Test + public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception { + final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0); + Injector injector; + + // init app + RMApp app = mock(RMApp.class); + when(app.getTrackingUrl()).thenReturn("http://host:123"); + when(app.getState()).thenReturn(RMAppState.FAILED); + when(app.getApplicationId()).thenReturn(APP_ID); + when(app.getApplicationType()).thenReturn("Type"); + when(app.getUser()).thenReturn("user"); + when(app.getName()).thenReturn("Name"); + when(app.getQueue()).thenReturn("queue"); + when(app.getDiagnostics()).thenReturn(new StringBuilder()); + when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED); + when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED); + when(app.getStartTime()).thenReturn(0L); + when(app.getFinishTime()).thenReturn(0L); + when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED); + + RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0); + when(app.getRMAppMetrics()).thenReturn(appMetrics); + + // initialize RM Context, and create RMApp, without creating RMAppAttempt + final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8); + rmContext.getRMApps().put(APP_ID, app); + + injector = + WebAppTests.createMockInjector(RMContext.class, rmContext, + new Module() { + @Override + public void configure(Binder binder) { + try { + binder.bind(ResourceManager.class).toInstance( + TestRMWebApp.mockRm(rmContext)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + }); + + AppBlock instance = injector.getInstance(AppBlock.class); + instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString()); + instance.render(); + } +}