From 426535007bcdc67331f7a37b5d69cc20b37c26e0 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Wed, 11 Mar 2015 19:35:19 -0700 Subject: [PATCH] YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM web page. Contributed by Xuan Gong. (cherry picked from commit 85f6d67fa78511f255fcfa810afc9a156a7b483b) --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/api/records/ContainerReport.java | 17 +++++++++++++- .../src/main/proto/yarn_protos.proto | 1 + .../yarn/client/cli/ApplicationCLI.java | 11 +++++++--- .../yarn/client/ProtocolHATestBase.java | 4 ++-- .../yarn/client/api/impl/TestAHSClient.java | 6 +++-- .../yarn/client/api/impl/TestYarnClient.java | 16 +++++++++----- .../hadoop/yarn/client/cli/TestYarnCLI.java | 18 +++++++++++---- .../impl/pb/ContainerReportPBImpl.java | 19 ++++++++++++++++ .../ApplicationHistoryManagerImpl.java | 2 +- ...licationHistoryManagerOnTimelineStore.java | 10 ++++++++- .../metrics/ContainerMetricsConstants.java | 2 ++ .../yarn/server/webapp/AppAttemptBlock.java | 12 +++++----- .../hadoop/yarn/server/webapp/AppBlock.java | 11 ++++------ .../yarn/server/webapp/ContainerBlock.java | 7 +++++- .../yarn/server/webapp/dao/ContainerInfo.java | 5 +++++ .../metrics/ContainerCreatedEvent.java | 8 ++++++- .../metrics/SystemMetricsPublisher.java | 5 ++++- .../rmcontainer/RMContainer.java | 1 + .../rmcontainer/RMContainerImpl.java | 22 +++++++++++++++++-- .../metrics/TestSystemMetricsPublisher.java | 5 ++++- 21 files changed, 147 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index dd6ce93c5c4..a2139631d5d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -707,6 +707,9 @@ Release 2.7.0 - UNRELEASED YARN-3295. Fix documentation nits found in markdown conversion. (Masatake Iwasaki via ozawa) + YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM + web page. (Xuan Gong via zjshen) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java index 4cce77f9b22..72b8edfe6f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.util.Records; *
  • {@link ContainerState} of the container.
    • *
  • Diagnostic information in case of errors.
    • *
  • Log URL.
    • + *
  • nodeHttpAddress
    • * *

    * @@ -54,7 +55,8 @@ public abstract class ContainerReport { public static ContainerReport newInstance(ContainerId containerId, Resource allocatedResource, NodeId assignedNode, Priority priority, long creationTime, long finishTime, String diagnosticInfo, String logUrl, - int containerExitStatus, ContainerState containerState) { + int containerExitStatus, ContainerState containerState, + String nodeHttpAddress) { ContainerReport report = Records.newRecord(ContainerReport.class); report.setContainerId(containerId); report.setAllocatedResource(allocatedResource); @@ -66,6 +68,7 @@ public abstract class ContainerReport { report.setLogUrl(logUrl); report.setContainerExitStatus(containerExitStatus); report.setContainerState(containerState); + report.setNodeHttpAddress(nodeHttpAddress); return report; } @@ -199,4 +202,16 @@ public abstract class ContainerReport { @Unstable public abstract void setContainerExitStatus(int containerExitStatus); + /** + * Get the Node Http address of the container + * + * @return the node http address of the container + */ + @Public + @Unstable + public abstract String getNodeHttpAddress(); + + @Private + @Unstable + public abstract void setNodeHttpAddress(String nodeHttpAddress); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 4e29d2fc82b..90706ed4be2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -98,6 +98,7 @@ message ContainerReportProto { optional string log_url = 8; optional int32 container_exit_status = 9; optional ContainerStateProto container_state = 10; + optional string node_http_address = 11; } enum YarnApplicationStateProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 108ad0b7220..dd4a949ed4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -63,7 +63,7 @@ public class ApplicationCLI extends YarnCLI { "%30s\t%20s\t%35s\t%35s" + System.getProperty("line.separator"); private static final String CONTAINER_PATTERN = - "%30s\t%20s\t%20s\t%20s\t%20s\t%35s" + "%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s" + System.getProperty("line.separator"); private static final String APP_TYPE_CMD = "appTypes"; @@ -355,6 +355,9 @@ public class ApplicationCLI extends YarnCLI { containerReportStr.println(containerReport.getLogUrl()); containerReportStr.print("\tHost : "); containerReportStr.println(containerReport.getAssignedNode()); + containerReportStr.print("\tNodeHttpAddress : "); + containerReportStr.println(containerReport.getNodeHttpAddress() == null + ? "N/A" : containerReport.getNodeHttpAddress()); containerReportStr.print("\tDiagnostics : "); containerReportStr.print(containerReport.getDiagnosticsInfo()); } else { @@ -595,7 +598,7 @@ public class ApplicationCLI extends YarnCLI { .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId)); writer.println("Total number of containers " + ":" + appsReport.size()); writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time", - "Finish Time", "State", "Host", "LOG-URL"); + "Finish Time", "State", "Host", "Node Http Address", "LOG-URL"); for (ContainerReport containerReport : appsReport) { writer.printf( CONTAINER_PATTERN, @@ -603,7 +606,9 @@ public class ApplicationCLI extends YarnCLI { Times.format(containerReport.getCreationTime()), Times.format(containerReport.getFinishTime()), containerReport.getContainerState(), containerReport - .getAssignedNode(), containerReport.getLogUrl()); + .getAssignedNode(), containerReport.getNodeHttpAddress() == null + ? "N/A" : containerReport.getNodeHttpAddress(), + containerReport.getLogUrl()); } writer.flush(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 782bc43e51a..f468bc115a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -77,7 +77,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; -import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -715,7 +714,8 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes { public ContainerReport createFakeContainerReport() { return ContainerReport.newInstance(createFakeContainerId(), null, NodeId.newInstance("localhost", 0), null, 1000l, 1200l, "", "", 0, - ContainerState.COMPLETE); + ContainerState.COMPLETE, + "http://" + NodeId.newInstance("localhost", 0).toString()); } public List createFakeContainerReports() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index a88189e5c0d..c3e3c414771 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -371,14 +371,16 @@ public class TestAHSClient { ContainerReport.newInstance( ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, - 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE, + "http://" + NodeId.newInstance("host", 2345).toString()); containerReports.add(container); ContainerReport container1 = ContainerReport.newInstance( ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, - 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE, + "http://" + NodeId.newInstance("host", 2345).toString()); containerReports.add(container1); containers.put(attempt.getApplicationAttemptId(), containerReports); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 9946506f413..de669f2098a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -117,7 +117,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.UTCClock; -import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -613,13 +612,15 @@ public class TestYarnClient { ContainerReport container = ContainerReport.newInstance( ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, - "diagnosticInfo", "logURL", 0, ContainerState.RUNNING); + "diagnosticInfo", "logURL", 0, ContainerState.RUNNING, + "http://" + NodeId.newInstance("host", 2345).toString()); containerReports.add(container); ContainerReport container1 = ContainerReport.newInstance( ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, - "diagnosticInfo", "logURL", 0, ContainerState.RUNNING); + "diagnosticInfo", "logURL", 0, ContainerState.RUNNING, + "http://" + NodeId.newInstance("host", 2345).toString()); containerReports.add(container1); containers.put(attempt.getApplicationAttemptId(), containerReports); @@ -630,18 +631,21 @@ public class TestYarnClient { container = ContainerReport.newInstance( ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, - "diagnosticInfo", "logURL", 0, null); + "diagnosticInfo", "logURL", 0, null, + "http://" + NodeId.newInstance("host", 2345).toString()); containerReportsForAHS.add(container); container1 = ContainerReport.newInstance( ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, - "diagnosticInfo", "HSlogURL", 0, null); + "diagnosticInfo", "HSlogURL", 0, null, + "http://" + NodeId.newInstance("host", 2345).toString()); containerReportsForAHS.add(container1); ContainerReport container2 = ContainerReport.newInstance( ContainerId.newContainerId(attempt.getApplicationAttemptId(),3), null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, - "diagnosticInfo", "HSlogURL", 0, ContainerState.COMPLETE); + "diagnosticInfo", "HSlogURL", 0, ContainerState.COMPLETE, + "http://" + NodeId.newInstance("host", 2345).toString()); containerReportsForAHS.add(container2); containersFromAHS.put(attempt.getApplicationAttemptId(), containerReportsForAHS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 088969f6b32..4b60c521673 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -224,7 +224,8 @@ public class TestYarnCLI { ContainerId containerId = ContainerId.newContainerId(attemptId, 1); ContainerReport container = ContainerReport.newInstance(containerId, null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678, - "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE, + "http://" + NodeId.newInstance("host", 2345).toString()); when(client.getContainerReport(any(ContainerId.class))).thenReturn( container); int result = cli.run(new String[] { "container", "-status", @@ -240,6 +241,7 @@ public class TestYarnCLI { pw.println("\tState : COMPLETE"); pw.println("\tLOG-URL : logURL"); pw.println("\tHost : host:1234"); + pw.println("\tNodeHttpAddress : http://host:2345"); pw.println("\tDiagnostics : diagnosticInfo"); pw.close(); String appReportStr = baos.toString("UTF-8"); @@ -259,13 +261,16 @@ public class TestYarnCLI { long time1=1234,time2=5678; ContainerReport container = ContainerReport.newInstance(containerId, null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2, - "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE, + "http://" + NodeId.newInstance("host", 2345).toString()); ContainerReport container1 = ContainerReport.newInstance(containerId1, null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2, - "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE); + "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE, + "http://" + NodeId.newInstance("host", 2345).toString()); ContainerReport container2 = ContainerReport.newInstance(containerId2, null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1,0, - "diagnosticInfo", "", 0, ContainerState.RUNNING); + "diagnosticInfo", "", 0, ContainerState.RUNNING, + "http://" + NodeId.newInstance("host", 2345).toString()); List reports = new ArrayList(); reports.add(container); reports.add(container1); @@ -273,6 +278,7 @@ public class TestYarnCLI { DateFormat dateFormat=new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy"); when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn( reports); + sysOutStream.reset(); int result = cli.run(new String[] { "container", "-list", attemptId.toString() }); assertEquals(0, result); @@ -285,24 +291,28 @@ public class TestYarnCLI { pw.print("\t Finish Time"); pw.print("\t State"); pw.print("\t Host"); + pw.print("\t Node Http Address"); pw.println("\t LOG-URL"); pw.print(" container_1234_0005_01_000001"); pw.print("\t"+dateFormat.format(new Date(time1))); pw.print("\t"+dateFormat.format(new Date(time2))); pw.print("\t COMPLETE"); pw.print("\t host:1234"); + pw.print("\t http://host:2345"); pw.println("\t logURL"); pw.print(" container_1234_0005_01_000002"); pw.print("\t"+dateFormat.format(new Date(time1))); pw.print("\t"+dateFormat.format(new Date(time2))); pw.print("\t COMPLETE"); pw.print("\t host:1234"); + pw.print("\t http://host:2345"); pw.println("\t logURL"); pw.print(" container_1234_0005_01_000003"); pw.print("\t"+dateFormat.format(new Date(time1))); pw.print("\t N/A"); pw.print("\t RUNNING"); pw.print("\t host:1234"); + pw.print("\t http://host:2345"); pw.println("\t "); pw.close(); String appReportStr = baos.toString("UTF-8"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java index 18c452f4999..1f0405f7274 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java @@ -338,4 +338,23 @@ public class ContainerReportPBImpl extends ContainerReport { ContainerStateProto containerState) { return ProtoUtils.convertFromProtoFormat(containerState); } + + @Override + public String getNodeHttpAddress() { + ContainerReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasNodeHttpAddress()) { + return null; + } + return (p.getNodeHttpAddress()); + } + + @Override + public void setNodeHttpAddress(String nodeHttpAddress) { + maybeInitBuilder(); + if (nodeHttpAddress == null) { + builder.clearNodeHttpAddress(); + return; + } + builder.setNodeHttpAddress(nodeHttpAddress); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 803dc01d1d2..c7cf07b1a88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -215,7 +215,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements containerHistory.getStartTime(), containerHistory.getFinishTime(), containerHistory.getDiagnosticsInfo(), logUrl, containerHistory.getContainerExitStatus(), - containerHistory.getContainerState()); + containerHistory.getContainerState(), null); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 22418a8535c..1010f62835e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -415,6 +415,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService String diagnosticsInfo = null; int exitStatus = ContainerExitStatus.INVALID; ContainerState state = null; + String nodeHttpAddress = null; Map entityInfo = entity.getOtherInfo(); if (entityInfo != null) { if (entityInfo @@ -444,6 +445,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService allocatedPriority = (Integer) entityInfo.get( ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO); } + if (entityInfo.containsKey( + ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO)) { + nodeHttpAddress = + (String) entityInfo + .get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO); + } } List events = entity.getEvents(); if (events != null) { @@ -493,7 +500,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService Resource.newInstance(allocatedMem, allocatedVcore), NodeId.newInstance(allocatedHost, allocatedPort), Priority.newInstance(allocatedPriority), - createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state); + createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state, + nodeHttpAddress); } private ApplicationReportExt generateApplicationReport(TimelineEntity entity, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java index 8791da4f219..0d5540df6bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java @@ -57,4 +57,6 @@ public class ContainerMetricsConstants { public static final String STATE_EVENT_INFO = "YARN_CONTAINER_STATE"; + public static final String ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO = + "YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index 1bba4d8c4a4..4a82c93614d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -199,12 +199,14 @@ public class AppAttemptBlock extends HtmlBlock { .append(url("container", container.getContainerId())) .append("'>") .append(container.getContainerId()) - .append("\",\"") - .append(container.getAssignedNodeId() == null ? "N/A" : + .append(container.getNodeHttpAddress() == null ? "N/A" : StringEscapeUtils.escapeJavaScript(StringEscapeUtils - .escapeHtml(container.getAssignedNodeId()))) + .escapeHtml(container.getNodeHttpAddress()))) .append("\",\"") .append(container.getContainerExitStatus()).append("\",\"") .append(nodeLink == null ? "N/A" : StringEscapeUtils .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink))) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java index ed50c7af671..cae8d2e6fb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java @@ -104,7 +104,12 @@ public class ContainerBlock extends HtmlBlock { container.getContainerState() == null ? UNAVAILABLE : container .getContainerState()) ._("Exit Status:", container.getContainerExitStatus()) - ._("Node:", container.getAssignedNodeId()) + ._( + "Node:", + container.getNodeHttpAddress() == null ? "#" : container + .getNodeHttpAddress(), + container.getNodeHttpAddress() == null ? "N/A" : container + .getNodeHttpAddress()) ._("Priority:", container.getPriority()) ._("Started:", Times.format(container.getStartedTime())) ._( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java index bdcc7b2e30e..0d18e7af475 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java @@ -42,6 +42,7 @@ public class ContainerInfo { protected String logUrl; protected int containerExitStatus; protected ContainerState containerState; + protected String nodeHttpAddress; public ContainerInfo() { // JAXB needs this @@ -64,6 +65,7 @@ public class ContainerInfo { logUrl = container.getLogUrl(); containerExitStatus = container.getContainerExitStatus(); containerState = container.getContainerState(); + nodeHttpAddress = container.getNodeHttpAddress(); } public String getContainerId() { @@ -114,4 +116,7 @@ public class ContainerInfo { return containerState; } + public String getNodeHttpAddress() { + return nodeHttpAddress; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java index eeda18199b5..05b67811138 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java @@ -29,18 +29,21 @@ public class ContainerCreatedEvent extends SystemMetricsEvent { private Resource allocatedResource; private NodeId allocatedNode; private Priority allocatedPriority; + private String nodeHttpAddress; public ContainerCreatedEvent( ContainerId containerId, Resource allocatedResource, NodeId allocatedNode, Priority allocatedPriority, - long createdTime) { + long createdTime, + String nodeHttpAddress) { super(SystemMetricsEventType.CONTAINER_CREATED, createdTime); this.containerId = containerId; this.allocatedResource = allocatedResource; this.allocatedNode = allocatedNode; this.allocatedPriority = allocatedPriority; + this.nodeHttpAddress = nodeHttpAddress; } @Override @@ -64,4 +67,7 @@ public class ContainerCreatedEvent extends SystemMetricsEvent { return allocatedPriority; } + public String getNodeHttpAddress() { + return nodeHttpAddress; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 3adf519a6c3..b849b0035db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -181,7 +181,7 @@ public class SystemMetricsPublisher extends CompositeService { container.getAllocatedResource(), container.getAllocatedNode(), container.getAllocatedPriority(), - createdTime)); + createdTime, container.getNodeHttpAddress())); } } @@ -388,6 +388,9 @@ public class SystemMetricsPublisher extends CompositeService { event.getAllocatedNode().getPort()); entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO, event.getAllocatedPriority().getPriority()); + entityInfo.put( + ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO, + event.getNodeHttpAddress()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java index 9e9dcb9aa6f..20087f5fc0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java @@ -79,4 +79,5 @@ public interface RMContainer extends EventHandler { List getResourceRequests(); + String getNodeHttpAddress(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index e37d8fd3c8f..38a03aef145 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -41,7 +41,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent; @@ -573,10 +572,29 @@ public class RMContainerImpl implements RMContainer { this.getAllocatedResource(), this.getAllocatedNode(), this.getAllocatedPriority(), this.getCreationTime(), this.getFinishTime(), this.getDiagnosticsInfo(), this.getLogURL(), - this.getContainerExitStatus(), this.getContainerState()); + this.getContainerExitStatus(), this.getContainerState(), + this.getNodeHttpAddress()); } finally { this.readLock.unlock(); } return containerReport; } + + @Override + public String getNodeHttpAddress() { + try { + readLock.lock(); + if (container.getNodeHttpAddress() != null) { + StringBuilder httpAddress = new StringBuilder(); + httpAddress.append(WebAppUtils.getHttpSchemePrefix(rmContext + .getYarnConfiguration())); + httpAddress.append(container.getNodeHttpAddress()); + return httpAddress.toString(); + } else { + return null; + } + } finally { + readLock.unlock(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 9f02721399a..7ed3835bdb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; -import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; @@ -386,6 +385,10 @@ public class TestSystemMetricsPublisher { when(container.getDiagnosticsInfo()).thenReturn("test diagnostics info"); when(container.getContainerExitStatus()).thenReturn(-1); when(container.getContainerState()).thenReturn(ContainerState.COMPLETE); + Container mockContainer = mock(Container.class); + when(container.getContainer()).thenReturn(mockContainer); + when(mockContainer.getNodeHttpAddress()) + .thenReturn("http://localhost:1234"); return container; }