diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 4ee9e404317..9dba9fc1d31 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -217,6 +217,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3679. AM logs and others should not automatically refresh after every 1 second. (Vinod KV via mahadev) + MAPREDUCE-3754. Modified RM UI to filter applications based on state of the + applications. (vinodkv) + OPTIMIZATIONS MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index fbc0887e129..eb492f02c75 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -25,4 +25,7 @@ public interface YarnWebParams { String CONTAINER_LOG_TYPE= "log.type"; String ENTITY_STRING = "entity.string"; String APP_OWNER = "app.owner"; + String APP_STATE = "app.state"; + String QUEUE_NAME = "queue.name"; + String NODE_STATE = "node.state"; } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index c629718a9be..9f4d0f94679 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -19,10 +19,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.util.StringHelper.sjoin; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; @@ -56,7 +59,12 @@ class AppsBlock extends HtmlBlock { th(".note", "Note")._()._(). tbody(); int i = 0; + String reqState = $(APP_STATE); + reqState = (reqState == null ? "" : reqState); for (RMApp app : list.apps.values()) { + if (!reqState.isEmpty() && app.getState() != RMAppState.valueOf(reqState)) { + continue; + } AppInfo appInfo = new AppInfo(app, true); String percent = String.format("%.1f", appInfo.getProgress()); tbody. @@ -86,7 +94,7 @@ class AppsBlock extends HtmlBlock { if (list.rendering == Render.JS_ARRAY) { echo("\n"); } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java index a7b35abaaac..222d6e49d5e 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java @@ -31,6 +31,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.webapp.Controller.RequestContext; import org.apache.hadoop.yarn.webapp.ToJSON; @@ -51,10 +52,14 @@ class AppsList implements ToJSON { apps = rmContext.getRMApps(); } - void toDataTableArrays(PrintWriter out) { + void toDataTableArrays(String requiredAppState, PrintWriter out) { out.append('['); boolean first = true; for (RMApp app : apps.values()) { + if (requiredAppState != null && !requiredAppState.isEmpty() + && app.getState() != RMAppState.valueOf(requiredAppState)) { + continue; + } AppInfo appInfo = new AppInfo(app, false); if (first) { first = false; @@ -84,7 +89,7 @@ class AppsList implements ToJSON { @Override public void toJSON(PrintWriter out) { out.print("{\"aaData\":"); - toDataTableArrays(out); + toDataTableArrays(null, out); out.print("}\n"); } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java index cf474b17d34..e6ba6130655 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java @@ -18,18 +18,32 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; public class NavBlock extends HtmlBlock { @Override public void render(Block html) { - html. + UL> mainList = html. div("#nav"). h3("Cluster"). ul(). li().a(url("cluster"), "About")._(). - li().a(url("nodes"), "Nodes")._(). - li().a(url("apps"), "Applications")._(). + li().a(url("nodes"), "Nodes")._(); + UL>>> subAppsList = mainList. + li().a(url("apps"), "Applications"). + ul(); + subAppsList.li()._(); + for (RMAppState state : RMAppState.values()) { + subAppsList. + li().a(url("apps", state.toString()), state.toString())._(); + } + subAppsList._()._(); + mainList. li().a(url("scheduler"), "Scheduler")._()._(). h3("Tools"). ul(). diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 74266a0015c..90b0824f0cc 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -25,14 +25,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; +import org.apache.hadoop.yarn.webapp.YarnWebParams; /** * The RM webapp */ -public class RMWebApp extends WebApp { - static final String APP_ID = "app.id"; - static final String QUEUE_NAME = "queue.name"; - static final String NODE_STATE = "node.state"; +public class RMWebApp extends WebApp implements YarnWebParams { private final ResourceManager rm; @@ -53,9 +51,9 @@ public class RMWebApp extends WebApp { } route("/", RmController.class); route(pajoin("/nodes", NODE_STATE), RmController.class, "nodes"); - route("/apps", RmController.class); + route(pajoin("/apps", APP_STATE), RmController.class); route("/cluster", RmController.class, "about"); - route(pajoin("/app", APP_ID), RmController.class, "app"); + route(pajoin("/app", APPLICATION_ID), RmController.class, "app"); route("/scheduler", RmController.class, "scheduler"); route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue"); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java index bb3ff674edc..c8778b87be8 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java @@ -18,9 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; -import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.APP_ID; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.QUEUE_NAME; import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID; import javax.servlet.http.HttpServletResponse; @@ -64,7 +64,7 @@ public class RmController extends Controller { } public void app() { - String aid = $(APP_ID); + String aid = $(APPLICATION_ID); if (aid.isEmpty()) { setStatus(HttpServletResponse.SC_BAD_REQUEST); setTitle("Bad request: requires application ID"); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java index 6900352463b..47532c1562d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout; +import static org.apache.hadoop.yarn.util.StringHelper.sjoin; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*; // Do NOT rename/refactor this to RMView as it will wreak havoc @@ -36,6 +38,11 @@ public class RmView extends TwoColumnLayout { set(DATATABLES_ID, "apps"); set(initID(DATATABLES, "apps"), appsTableInit()); setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}"); + + // Set the correct title. + String reqState = $(APP_STATE); + reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState); + setTitle(sjoin(reqState, "Applications")); } protected void commonPreHead(Page.HTML<_> html) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 659bf55a5e0..fd41d91f6a6 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -22,14 +22,14 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationMaster; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.ApplicationStatus; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -41,7 +41,6 @@ import com.google.common.collect.Lists; @InterfaceAudience.Private public abstract class MockAsm extends MockApps { - static final int DT = 1000000; // ms public static class AppMasterBase implements ApplicationMaster { @Override @@ -232,9 +231,10 @@ public abstract class MockAsm extends MockApps { final String user = newUserName(); final String name = newAppName(); final String queue = newQueue(); - final long start = System.currentTimeMillis() - (int)(Math.random()*DT); - final long finish = Math.random() < 0.5 ? 0 : - System.currentTimeMillis() + (int)(Math.random()*DT); + final long start = 123456 + i * 1000; + final long finish = 234567 + i * 1000; + RMAppState[] allStates = RMAppState.values(); + final RMAppState state = allStates[i % allStates.length]; return new ApplicationBase() { @Override public ApplicationId getApplicationId() { @@ -270,7 +270,7 @@ public abstract class MockAsm extends MockApps { } @Override public RMAppState getState() { - return RMAppState.RUNNING; + return state; } @Override public StringBuilder getDiagnostics() { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 5a492340d00..73554b0e595 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.webapp.WebApps; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.junit.Test; @@ -74,7 +76,7 @@ public class TestRMWebApp { @Test public void testView() { Injector injector = WebAppTests.createMockInjector(RMContext.class, - mockRMContext(3, 1, 2, 8*GiB), + mockRMContext(15, 1, 2, 8*GiB), new Module() { @Override public void configure(Binder binder) { @@ -85,7 +87,9 @@ public class TestRMWebApp { } } }); - injector.getInstance(RmView.class).render(); + RmView rmViewInstance = injector.getInstance(RmView.class); + rmViewInstance.set(YarnWebParams.APP_STATE, RMAppState.RUNNING.toString()); + rmViewInstance.render(); WebAppTests.flushOutput(injector); }