Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created. Contributed by Wangda Tan 

(cherry picked from commit 81c9d17af8)

(cherry picked from commit 242fd0e39a)
(cherry picked from commit a9d5acd898b34e1050a78f2d70ed62fdb82948a6)
This commit is contained in:
Jason Lowe 2014-11-17 21:13:22 +00:00 committed by Vinod Kumar Vavilapalli
parent f83d898944
commit f307b426f3
5 changed files with 130 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -18,6 +18,9 @@ Release 2.6.1 - UNRELEASED
YARN-2816. NM fail to start with NPE during container recovery (Zhihai Xu
via jlowe)
YARN-2414. RM web UI: app page will crash if app is failed before any
attempt has been created (Wangda Tan via jlowe)
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
View File

@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import com.google.common.annotations.VisibleForTesting;
@InterfaceAudience.Private
public class ApplicationACLsManager {
@ -48,6 +50,11 @@ public class ApplicationACLsManager {
private final ConcurrentMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>> applicationACLS
= new ConcurrentHashMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>>();
@VisibleForTesting
public ApplicationACLsManager() {
this(new Configuration());
}
public ApplicationACLsManager(Configuration conf) {
this.conf = conf;
this.adminAclsManager = new AdminACLsManager(this.conf);

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
View File

@ -24,9 +24,16 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import com.google.common.annotations.VisibleForTesting;
public class QueueACLsManager {
private ResourceScheduler scheduler;
private boolean isACLsEnable;
@VisibleForTesting
public QueueACLsManager() {
this(null, new Configuration());
}
public QueueACLsManager(ResourceScheduler scheduler, Configuration conf) {
this.scheduler = scheduler;

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
View File

@ -33,6 +33,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@ -113,8 +115,23 @@ public class AppBlock extends HtmlBlock {
setTitle(join("Application ", aid));
RMAppMetrics appMerics = rmApp.getRMAppMetrics();
RMAppAttemptMetrics attemptMetrics =
rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
// Get attempt metrics and fields, it is possible currentAttempt of RMApp is
// null. In that case, we will assume resource preempted and number of Non
// AM container preempted on that attempt is 0
RMAppAttemptMetrics attemptMetrics;
if (null == rmApp.getCurrentAppAttempt()) {
attemptMetrics = null;
} else {
attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
}
Resource attemptResourcePreempted =
attemptMetrics == null ? Resources.none() : attemptMetrics
.getResourcePreempted();
int attemptNumNonAMContainerPreempted =
attemptMetrics == null ? 0 : attemptMetrics
.getNumNonAMContainersPreempted();
info("Application Overview")
._("User:", app.getUser())
._("Name:", app.getName())
@ -143,13 +160,12 @@ public class AppBlock extends HtmlBlock {
._("Total Number of AM Containers Preempted:",
String.valueOf(appMerics.getNumAMContainersPreempted()))
._("Resource Preempted from Current Attempt:",
attemptMetrics.getResourcePreempted())
attemptResourcePreempted)
._("Number of Non-AM Containers Preempted from Current Attempt:",
String.valueOf(attemptMetrics
.getNumNonAMContainersPreempted()))
attemptNumNonAMContainerPreempted)
._("Aggregate Resource Allocation:",
String.format("%d MB-seconds, %d vcore-seconds",
appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
String.format("%d MB-seconds, %d vcore-seconds",
appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
pdiv._();
Collection<RMAppAttempt> attempts = rmApp.getAppAttempts().values();

90
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java Normal file
View File

@ -0,0 +1,90 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Test;
import com.google.inject.Binder;
import com.google.inject.Injector;
import com.google.inject.Module;
public class TestAppPage {
@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
Injector injector;
// init app
RMApp app = mock(RMApp.class);
when(app.getTrackingUrl()).thenReturn("http://host:123");
when(app.getState()).thenReturn(RMAppState.FAILED);
when(app.getApplicationId()).thenReturn(APP_ID);
when(app.getApplicationType()).thenReturn("Type");
when(app.getUser()).thenReturn("user");
when(app.getName()).thenReturn("Name");
when(app.getQueue()).thenReturn("queue");
when(app.getDiagnostics()).thenReturn(new StringBuilder());
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
when(app.getStartTime()).thenReturn(0L);
when(app.getFinishTime()).thenReturn(0L);
when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
rmContext.getRMApps().put(APP_ID, app);
injector =
WebAppTests.createMockInjector(RMContext.class, rmContext,
new Module() {
@Override
public void configure(Binder binder) {
try {
binder.bind(ResourceManager.class).toInstance(
TestRMWebApp.mockRm(rmContext));
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
});
AppBlock instance = injector.getInstance(AppBlock.class);
instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
instance.render();
}
}