YARN-3543. ApplicationReport should be able to tell whether the

Application is AM managed or not. Contributed by Rohith Sharma K S

(cherry picked from commit 0306d902f5)
This commit is contained in:
Xuan 2015-08-03 15:46:00 -07:00
parent b264217bc3
commit 8126fde884
21 changed files with 136 additions and 25 deletions

View File

@ -311,6 +311,9 @@ Release 2.8.0 - UNRELEASED
YARN-3965. Add startup timestamp to nodemanager UI (Hong Zhiguo via jlowe)
YARN-3543. ApplicationReport should be able to tell whether the Application
is AM managed or not. (Rohith Sharma K S via xgong)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

View File

@ -92,13 +92,14 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken,
Set<String> tags) {
Set<String> tags, boolean unmanagedApplication) {
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
finishTime, finalStatus, appResources, origTrackingUrl, progress,
applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
return report;
}
@ -393,4 +394,18 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
@Unstable
public abstract void setLogAggregationStatus(
LogAggregationStatus logAggregationStatus);
/**
* @return true if the AM is not managed by the RM
*/
@Public
@Unstable
public abstract boolean isUnmanagedApp();
/**
* @param value true if RM should not manage the AM
*/
@Public
@Unstable
public abstract void setUnmanagedApp(boolean unmanagedApplication);
}

View File

@ -195,6 +195,7 @@ message ApplicationReportProto {
optional hadoop.common.TokenProto am_rm_token = 19;
repeated string applicationTags = 20;
optional LogAggregationStatusProto log_aggregation_status = 21;
optional bool unmanaged_application = 22 [default = false];
}
enum LogAggregationStatusProto {

View File

@ -534,7 +534,9 @@ private int printApplicationReport(String applicationId)
appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A"
: appReport.getLogAggregationStatus());
appReportStr.print("\tDiagnostics : ");
appReportStr.print(appReport.getDiagnostics());
appReportStr.println(appReport.getDiagnostics());
appReportStr.print("\tUnmanaged Application : ");
appReportStr.print(appReport.isUnmanagedApp());
} else {
appReportStr.print("Application with id '" + applicationId
+ "' doesn't exist in RM.");

View File

@ -895,12 +895,12 @@ public void testAMMRTokens() throws Exception {
rmClient.start();
ApplicationId appId = createApp(rmClient, false);
waitTillAccepted(rmClient, appId);
waitTillAccepted(rmClient, appId, false);
//managed AMs don't return AMRM token
Assert.assertNull(rmClient.getAMRMToken(appId));
appId = createApp(rmClient, true);
waitTillAccepted(rmClient, appId);
waitTillAccepted(rmClient, appId, true);
long start = System.currentTimeMillis();
while (rmClient.getAMRMToken(appId) == null) {
if (System.currentTimeMillis() - start > 20 * 1000) {
@ -921,7 +921,7 @@ public ApplicationId run() throws Exception {
rmClient.init(yarnConf);
rmClient.start();
ApplicationId appId = createApp(rmClient, true);
waitTillAccepted(rmClient, appId);
waitTillAccepted(rmClient, appId, true);
long start = System.currentTimeMillis();
while (rmClient.getAMRMToken(appId) == null) {
if (System.currentTimeMillis() - start > 20 * 1000) {
@ -981,7 +981,8 @@ private ApplicationId createApp(YarnClient rmClient, boolean unmanaged)
return appId;
}
private void waitTillAccepted(YarnClient rmClient, ApplicationId appId)
private void waitTillAccepted(YarnClient rmClient, ApplicationId appId,
boolean unmanagedApplication)
throws Exception {
try {
long start = System.currentTimeMillis();
@ -994,6 +995,7 @@ private void waitTillAccepted(YarnClient rmClient, ApplicationId appId)
Thread.sleep(200);
report = rmClient.getApplicationReport(appId);
}
Assert.assertEquals(unmanagedApplication, report.isUnmanagedApp());
} catch (Exception ex) {
throw new Exception(ex);
}

View File

@ -104,7 +104,7 @@ public void testGetApplicationReport() throws Exception {
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
null);
null, null, false);
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
@ -131,6 +131,7 @@ public void testGetApplicationReport() throws Exception {
(i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
pw.println("\tLog Aggregation Status : SUCCEEDED");
pw.println("\tDiagnostics : diagnostics");
pw.println("\tUnmanaged Application : false");
pw.close();
String appReportStr = baos.toString("UTF-8");
Assert.assertEquals(appReportStr, sysOutStream.toString());

View File

@ -581,4 +581,16 @@ private LogAggregationStatus convertFromProtoFormat(
convertToProtoFormat(LogAggregationStatus s) {
return ProtoUtils.convertToProtoFormat(s);
}
@Override
public boolean isUnmanagedApp() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
return p.getUnmanagedApplication();
}
@Override
public void setUnmanagedApp(boolean unmanagedApplication) {
maybeInitBuilder();
builder.setUnmanagedApplication(unmanagedApplication);
}
}

View File

@ -58,7 +58,8 @@ protected static ApplicationReport createApplicationReport(
ApplicationReport.newInstance(appId, appAttemptId, "user", "queue",
"appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
"N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
"N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
null,false);
return appReport;
}

View File

@ -247,6 +247,7 @@ private static ApplicationReportExt convertToApplicationReport(
String queue = null;
String name = null;
String type = null;
boolean unmanagedApplication = false;
long createdTime = 0;
long finishedTime = 0;
float progress = 0.0f;
@ -277,7 +278,8 @@ private static ApplicationReportExt convertToApplicationReport(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
null, progress, type, null, appTags), appViewACLs);
null, progress, type, null, appTags,
unmanagedApplication), appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@ -294,6 +296,13 @@ private static ApplicationReportExt convertToApplicationReport(
entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO)
.toString();
}
if (entityInfo
.containsKey(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)) {
unmanagedApplication =
Boolean.parseBoolean(entityInfo.get(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)
.toString());
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_CPU_METRICS).toString());
@ -365,7 +374,7 @@ private static ApplicationReportExt convertToApplicationReport(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
null, progress, type, null, appTags), appViewACLs);
null, progress, type, null, appTags, unmanagedApplication), appViewACLs);
}
private static ApplicationAttemptReport convertToApplicationAttemptReport(

View File

@ -475,6 +475,8 @@ private static TimelineEntity createApplicationTimelineEntity(
"test app type");
entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1");
entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
entityInfo.put(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false");
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123);

View File

@ -74,4 +74,7 @@ public class ApplicationMetricsConstants {
"YARN_APPLICATION_LATEST_APP_ATTEMPT";
public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS";
public static final String UNMANAGED_APPLICATION_ENTITY_INFO =
"YARN_APPLICATION_UNMANAGED_APPLICATION";
}

View File

@ -206,6 +206,7 @@ public ApplicationReport run() throws Exception {
}
overviewTable._("Diagnostics:",
app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo());
overviewTable._("Unmanaged Application:", app.isUnmanagedApp());
Collection<ApplicationAttemptReport> attempts;
try {

View File

@ -60,6 +60,7 @@ public class AppInfo {
protected String applicationTags;
private int allocatedCpuVcores;
private int allocatedMemoryMB;
protected boolean unmanagedApplication;
public AppInfo() {
// JAXB needs this
@ -99,6 +100,7 @@ public AppInfo(ApplicationReport app) {
if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
}
unmanagedApplication = app.isUnmanagedApp();
}
public String getAppId() {
@ -188,4 +190,8 @@ public long getElapsedTime() {
public String getApplicationTags() {
return applicationTags;
}
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
}

View File

@ -32,6 +32,7 @@ public class ApplicationCreatedEvent extends
private String queue;
private long submittedTime;
private Set<String> appTags;
private boolean unmanagedApplication;
public ApplicationCreatedEvent(ApplicationId appId,
String name,
@ -40,7 +41,8 @@ public ApplicationCreatedEvent(ApplicationId appId,
String queue,
long submittedTime,
long createdTime,
Set<String> appTags) {
Set<String> appTags,
boolean unmanagedApplication) {
super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId;
this.name = name;
@ -49,6 +51,7 @@ public ApplicationCreatedEvent(ApplicationId appId,
this.queue = queue;
this.submittedTime = submittedTime;
this.appTags = appTags;
this.unmanagedApplication = unmanagedApplication;
}
@Override
@ -83,4 +86,8 @@ public long getSubmittedTime() {
public Set<String> getAppTags() {
return appTags;
}
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
}

View File

@ -106,7 +106,8 @@ public void appCreated(RMApp app, long createdTime) {
app.getUser(),
app.getQueue(),
app.getSubmitTime(),
createdTime, app.getApplicationTags()));
createdTime, app.getApplicationTags(),
app.getApplicationSubmissionContext().getUnmanagedAM()));
}
}
@ -253,6 +254,9 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
event.getSubmittedTime());
entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
event.getAppTags());
entityInfo.put(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
event.isUnmanagedApp());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(

View File

@ -663,8 +663,9 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName,
createApplicationState(), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
appUsageReport, origTrackingUrl, progress, this.applicationType,
amrmToken, applicationTags);
amrmToken, applicationTags);
report.setLogAggregationStatus(logAggregationStatus);
report.setUnmanagedApp(submissionContext.getUnmanagedAM());
return report;
} finally {
this.readLock.unlock();

View File

@ -96,6 +96,7 @@ public class AppInfo {
protected List<ResourceRequest> resourceRequests;
protected LogAggregationStatus logAggregationStatus;
protected boolean unmanagedApplication;
public AppInfo() {
} // JAXB needs this
@ -183,6 +184,8 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
appMetrics.getResourcePreempted().getVirtualCores();
memorySeconds = appMetrics.getMemorySeconds();
vcoreSeconds = appMetrics.getVcoreSeconds();
unmanagedApplication =
app.getApplicationSubmissionContext().getUnmanagedAM();
}
}
@ -321,4 +324,8 @@ public List<ResourceRequest> getResourceRequests() {
public LogAggregationStatus getLogAggregationStatus() {
return this.logAggregationStatus;
}
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
}

View File

@ -29,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@ -134,6 +135,12 @@ public void testPublishApplicationMetrics() throws Exception {
Assert.assertEquals(app.getQueue(),
entity.getOtherInfo()
.get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO));
Assert.assertEquals(
app.getApplicationSubmissionContext().getUnmanagedAM(),
entity.getOtherInfo().get(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO));
Assert
.assertEquals(
app.getUser(),
@ -362,6 +369,9 @@ private static RMApp createRMApp(ApplicationId appId) {
appTags.add("test");
appTags.add("tags");
when(app.getApplicationTags()).thenReturn(appTags);
ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class);
when(asc.getUnmanagedAM()).thenReturn(false);
when(app.getApplicationSubmissionContext()).thenReturn(asc);
return app;
}

View File

@ -1308,14 +1308,15 @@ public void verifyAppsXML(NodeList nodes, RMApp app) throws JSONException,
WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"),
WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"),
WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"),
WebServicesTestUtils.getXmlString(element, "logAggregationStatus"));
WebServicesTestUtils.getXmlString(element, "logAggregationStatus"),
WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication"));
}
}
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 28, info.length());
assertEquals("incorrect number of elements", 29, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"),
@ -1331,7 +1332,8 @@ public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
info.getInt("preemptedResourceVCores"),
info.getInt("numNonAMContainerPreempted"),
info.getInt("numAMContainerPreempted"),
info.getString("logAggregationStatus"));
info.getString("logAggregationStatus"),
info.getBoolean("unmanagedApplication"));
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
@ -1342,7 +1344,8 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
int allocatedMB, int allocatedVCores, int numContainers,
int preemptedResourceMB, int preemptedResourceVCores,
int numNonAMContainerPreempted, int numAMContainerPreempted,
String logAggregationStatus) throws JSONException,
String logAggregationStatus, boolean unmanagedApplication)
throws JSONException,
Exception {
WebServicesTestUtils.checkStringMatch("id", app.getApplicationId()
@ -1392,6 +1395,9 @@ public void verifyAppInfoGeneric(RMApp app, String id, String user,
assertEquals("Log aggregation Status doesn't match", app
.getLogAggregationStatusForAppReport().toString(),
logAggregationStatus);
assertEquals("unmanagedApplication doesn't match", app
.getApplicationSubmissionContext().getUnmanagedAM(),
unmanagedApplication);
}
@Test

View File

@ -1380,7 +1380,8 @@ Response Body:
"applicationType" : "MAPREDUCE",
"applicationTags" : ""
"memorySeconds" : 151730,
"vcoreSeconds" : 103
"vcoreSeconds" : 103,
"unmanagedApplication":"false"
},
{
"finishedTime" : 1326815789546,
@ -1406,7 +1407,8 @@ Response Body:
"applicationTags" : "tag1"
"memorySeconds" : 640064,
"vcoreSeconds" : 442
}
"unmanagedApplication":"false"
}
]
}
}
@ -1455,6 +1457,7 @@ Response Body:
<runningContainers>0</runningContainers>
<memorySeconds>151730</memorySeconds>
<vcoreSeconds>103</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
</app>
<app>
<id>application_1326815542473_0002</id>
@ -1480,6 +1483,7 @@ Response Body:
<runningContainers>0</runningContainers>
<memorySeconds>640064</memorySeconds>
<vcoreSeconds>442</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
</app>
</apps>
```
@ -1639,6 +1643,7 @@ Note that depending on security settings a user might not be able to see all the
| runningContainers | int | The number of containers currently running for the application |
| memorySeconds | long | The amount of memory the application has allocated (megabyte-seconds) |
| vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) |
| unmanagedApplication | boolean | Is the application unmanaged. |
### Response Examples
@ -1678,7 +1683,8 @@ Response Body:
"trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5",
"queue" : "a1",
"memorySeconds" : 151730,
"vcoreSeconds" : 103
"vcoreSeconds" : 103,
"unmanagedApplication":"false"
}
}
```
@ -1720,6 +1726,7 @@ Response Body:
<amHostHttpAddress>host.domain.com:8042</amHostHttpAddress>
<memorySeconds>151730</memorySeconds>
<vcoreSeconds>103</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
</app>
```

View File

@ -1087,7 +1087,8 @@ Response Body:
"submittedTime":1430425001004,
"startedTime":1430425001004,
"finishedTime":1430425008861,
"elapsedTime":7857},
"elapsedTime":7857,
"unmanagedApplication":"false"},
{
"appId":"application_1430424020775_0003",
"currentAppAttemptId":"appattempt_1430424020775_0003_000001",
@ -1106,7 +1107,8 @@ Response Body:
"submittedTime":1430424956650,
"startedTime":1430424956650,
"finishedTime":1430424963907,
"elapsedTime":7257},
"elapsedTime":7257,
"unmanagedApplication":"false"},
{
"appId":"application_1430424020775_0002",
"currentAppAttemptId":"appattempt_1430424020775_0002_000001",
@ -1125,7 +1127,8 @@ Response Body:
"submittedTime":1430424769395,
"startedTime":1430424769395,
"finishedTime":1430424776594,
"elapsedTime":7199},
"elapsedTime":7199,
"unmanagedApplication":"false"},
{
"appId":"application_1430424020775_0001",
"currentAppAttemptId":"appattempt_1430424020775_0001_000001",
@ -1145,7 +1148,8 @@ Response Body:
"startedTime":1430424072153,
"finishedTime":1430424776594,
"elapsedTime":18344,
"applicationTags":"mrapplication,ta-example"
"applicationTags":"mrapplication,ta-example",
"unmanagedApplication":"false"
}
]
}
@ -1187,6 +1191,7 @@ Response Body:
<startedTime>1430425001004</startedTime>
<finishedTime>1430425008861</finishedTime>
<elapsedTime>7857</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
</app>
<app>
<appId>application_1430424020775_0003</appId>
@ -1207,6 +1212,7 @@ Response Body:
<startedTime>1430424956650</startedTime>
<finishedTime>1430424963907</finishedTime>
<elapsedTime>7257</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
</app>
<app>
<appId>application_1430424020775_0002</appId>
@ -1227,6 +1233,7 @@ Response Body:
<startedTime>1430424769395</startedTime>
<finishedTime>1430424776594</finishedTime>
<elapsedTime>7199</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
</app>
<app>
<appId>application_1430424020775_0001</appId>
@ -1248,6 +1255,7 @@ Response Body:
<finishedTime>1430424072153</finishedTime>
<elapsedTime>18344</elapsedTime>
<applicationTags>mrapplication,ta-example</applicationTags>
<unmanagedApplication>false</unmanagedApplication>
</app>
</apps>
@ -1298,6 +1306,7 @@ None
| `host` | string | The host of the ApplicationMaster |
| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared |
| `applicationTags` | string | The application tags. |
| `unmanagedApplication` | boolean | Is the application unmanaged. |
### Response Examples:
@ -1334,7 +1343,8 @@ Response Body:
"startedTime": 1430424053809,
"finishedTime": 1430424072153,
"elapsedTime": 18344,
"applicationTags": mrapplication,tag-example
"applicationTags": mrapplication,tag-example,
"unmanagedApplication":"false"
}
#### XML response
@ -1373,6 +1383,7 @@ Response Body:
<finishedTime>1430424072153</finishedTime>
<elapsedTime>18344</elapsedTime>
<applicationTags>mrapplication,ta-example</applicationTags>
<unmanagedApplication>false</unmanagedApplication>
</app>
## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List