From ef59ffd362b9a91be08cbdbaa15aafdf08f00bdc Mon Sep 17 00:00:00 2001 From: Giovanni Matteo Fumarola Date: Thu, 19 Dec 2019 11:37:17 -0800 Subject: [PATCH] YARN-10038. [UI] Finish Time is not correctly parsed in the RM Apps page. Contributed by Inigo Goiri. --- .../yarn/server/webapp/WebPageUtils.java | 28 +++++----- .../resourcemanager/webapp/ColumnHeader.java | 48 +++++++++++++++++ .../resourcemanager/webapp/RMAppsBlock.java | 54 ++++++++++++------- .../resourcemanager/webapp/TestRMWebApp.java | 37 +++++++++++++ 4 files changed, 135 insertions(+), 32 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ColumnHeader.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index 06a5f1df144..cf4e020d35e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -33,8 +33,10 @@ public class WebPageUtils { public static String appsTableInit( boolean isFairSchedulerPage, boolean isResourceManager) { - // id, user, name, queue, starttime, finishtime, state, status, progress, ui + // id, user, name, app type, app tags, queue, priority, + // starttime, launchtime, finishtime, state, status, progress, ui // FairSchedulerPage's table is a bit different + // This is define in RMAppsBlock.COLUMNS for the RM return tableInit() .append(", 'aaData': appsTableData") .append(", bDeferRender: true") @@ -51,24 +53,26 @@ public class WebPageUtils { String progressIndex = "[11]"; StringBuilder sb = new StringBuilder(); sb.append("[\n") - .append("{'sType':'natural', 'aTargets': [0]") - .append(", 'mRender': parseHadoopID }") - .append("\n, {'sType':'num-ignore-str', 'aTargets': [6, 7, 8]") - .append(", 'mRender': renderHadoopDate }"); + .append("{'sType':'natural', 'aTargets': [0], ") + .append("'mRender': parseHadoopID },\n") + .append("{'sType':'num-ignore-str', 'aTargets': [7, 8, 9], ") + .append("'mRender': renderHadoopDate },\n"); if (isResourceManager) { // Update following line if any column added in RM page before column 11 - sb.append("\n, {'sType':'num-ignore-str', 'aTargets': [11, 12, 13, 14, 15] }"); - // set progress column index to 18 - progressIndex = "[18]"; + sb.append("{'sType':'num-ignore-str', ") + .append("'aTargets': [12, 13, 14, 15, 16] },\n"); + // set progress column index to 19 + progressIndex = "[19]"; } else if (isFairSchedulerPage) { // Update following line if any column added in scheduler page before column 11 - sb.append("\n, {'sType':'num-ignore-str', 'aTargets': [11, 12, 13, 14, 15] }"); + sb.append("{'sType':'num-ignore-str', ") + .append("'aTargets': [11, 12, 13, 14, 15] },\n"); // set progress column index to 16 progressIndex = "[16]"; } - sb.append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':"); - sb.append(progressIndex); - sb.append(", 'mRender': parseHadoopProgress }]"); + sb.append("{'sType':'numeric', bSearchable:false, 'aTargets':") + .append(progressIndex) + .append(", 'mRender': parseHadoopProgress }\n]"); return sb.toString(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ColumnHeader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ColumnHeader.java new file mode 100644 index 00000000000..a251c98f6f3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/ColumnHeader.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +/** + * Header for a Web UI column. This used with TH. + */ +public class ColumnHeader { + private String selector; + private String cdata; + + public ColumnHeader(String pselector, String pcdata) { + this.selector = pselector; + this.cdata = pcdata; + } + + /** + * Get the selector field for the TH. + * @return Selector. + */ + public String getSelector() { + return this.selector; + } + + /** + * Get the cdata field for the TH. + * @return CData. + */ + public String getCData() { + return this.cdata; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index 3d257b774f8..8868fa10935 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.webapp.View; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.THEAD; +import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR; import com.google.inject.Inject; @@ -49,6 +51,32 @@ public class RMAppsBlock extends AppsBlock { private ResourceManager rm; + /** Columns for the Apps RM page. */ + static final ColumnHeader[] COLUMNS = { + new ColumnHeader(".id", "ID"), + new ColumnHeader(".user", "User"), + new ColumnHeader(".name", "Name"), + new ColumnHeader(".type", "Application Type"), + new ColumnHeader(".apptag", "Application Tags"), + new ColumnHeader(".queue", "Queue"), + new ColumnHeader(".priority", "Application Priority"), + new ColumnHeader(".starttime", "StartTime"), + new ColumnHeader(".IDlaunchtime", "LaunchTime"), + new ColumnHeader(".finishtime", "FinishTime"), + new ColumnHeader(".state", "State"), + new ColumnHeader(".finalstatus", "FinalStatus"), + new ColumnHeader(".runningcontainer", "Running Containers"), + new ColumnHeader(".allocatedCpu", "Allocated CPU VCores"), + new ColumnHeader(".allocatedMemory", "Allocated Memory MB"), + new ColumnHeader(".reservedCpu", "Reserved CPU VCores"), + new ColumnHeader(".reservedMemory", "Reserved Memory MB"), + new ColumnHeader(".queuePercentage", "% of Queue"), + new ColumnHeader(".clusterPercentage", "% of Cluster"), + new ColumnHeader(".progress", "Progress"), + new ColumnHeader(".ui", "Tracking UI"), + new ColumnHeader(".blacklisted", "Blacklisted Nodes"), + }; + @Inject RMAppsBlock(ResourceManager rm, View.ViewContext ctx) { super(null, ctx); @@ -57,26 +85,12 @@ public class RMAppsBlock extends AppsBlock { @Override protected void renderData(Block html) { - TBODY> tbody = - html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") - .th(".name", "Name").th(".type", "Application Type") - .th(".apptag", "Application Tags") - .th(".queue", "Queue").th(".priority", "Application Priority") - .th(".starttime", "StartTime") - .th("launchtime", "LaunchTime") - .th(".finishtime", "FinishTime").th(".state", "State") - .th(".finalstatus", "FinalStatus") - .th(".runningcontainer", "Running Containers") - .th(".allocatedCpu", "Allocated CPU VCores") - .th(".allocatedMemory", "Allocated Memory MB") - .th(".reservedCpu", "Reserved CPU VCores") - .th(".reservedMemory", "Reserved Memory MB") - .th(".queuePercentage", "% of Queue") - .th(".clusterPercentage", "% of Cluster") - .th(".progress", "Progress") - .th(".ui", "Tracking UI") - .th(".blacklisted", "Blacklisted Nodes").__() - .__().tbody(); + + TR>> tr = html.table("#apps").thead().tr(); + for (ColumnHeader col : COLUMNS) { + tr = tr.th(col.getSelector(), col.getCData()); + } + TBODY> tbody = tr.__().__().tbody(); StringBuilder appsTableData = new StringBuilder("[\n"); for (ApplicationReport appReport : appReports) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index e17100eeb05..1a4d6e3a6c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -21,12 +21,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.server.resourcemanager.MockNodes.newResource; import static org.apache.hadoop.yarn.webapp.Params.TITLE; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentMap; @@ -62,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSec import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.server.webapp.WebPageUtils; import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.YarnWebParams; @@ -159,6 +162,40 @@ public class TestRMWebApp { } + @Test + public void testRMAppColumnIndices() { + + // Find the columns to check + List colsId = new LinkedList(); + List colsTime = new LinkedList(); + List colsProgress = new LinkedList(); + for (int i = 0; i < RMAppsBlock.COLUMNS.length; i++) { + ColumnHeader col = RMAppsBlock.COLUMNS[i]; + if (col.getCData().contains("ID")) { + colsId.add(i); + } else if (col.getCData().contains("Time")) { + colsTime.add(i); + } else if (col.getCData().contains("Progress")) { + colsProgress.add(i); + } + } + + // Verify that the table JS header matches the columns + String tableInit = WebPageUtils.appsTableInit(true); + for (String tableLine : tableInit.split("\\n")) { + if (tableLine.contains("parseHadoopID")) { + assertTrue(tableLine + " should have id " + colsId, + tableLine.contains(colsId.toString())); + } else if (tableLine.contains("renderHadoopDate")) { + assertTrue(tableLine + " should have dates " + colsTime, + tableLine.contains(colsTime.toString())); + } else if (tableLine.contains("parseHadoopProgress")) { + assertTrue(tableLine + " should have progress " + colsProgress, + tableLine.contains(colsProgress.toString())); + } + } + } + public static RMContext mockRMContext(int numApps, int racks, int numNodes, int mbsPerNode) { final List apps = MockAsm.newApplications(numApps);