YARN-1462. AHS API and other AHS changes to handle tags for completed MR jobs. Contributed by Xuan Gong
This commit is contained in:
parent
75885852cc
commit
3e000a919f
|
@ -292,6 +292,8 @@ Release 2.8.0 - UNRELEASED
|
|||
YARN-2392. Add more diags about app retry limits on AM failures. (Steve
|
||||
Loughran via jianhe)
|
||||
|
||||
YARN-1462. AHS API and other AHS changes to handle tags for completed MR jobs. (xgong)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
YARN-3339. TestDockerContainerExecutor should pull a single image and not
|
||||
|
|
|
@ -83,6 +83,25 @@ public abstract class ApplicationReport {
|
|||
return report;
|
||||
}
|
||||
|
||||
@Private
|
||||
@Unstable
|
||||
public static ApplicationReport newInstance(ApplicationId applicationId,
|
||||
ApplicationAttemptId applicationAttemptId, String user, String queue,
|
||||
String name, String host, int rpcPort, Token clientToAMToken,
|
||||
YarnApplicationState state, String diagnostics, String url,
|
||||
long startTime, long finishTime, FinalApplicationStatus finalStatus,
|
||||
ApplicationResourceUsageReport appResources, String origTrackingUrl,
|
||||
float progress, String applicationType, Token amRmToken,
|
||||
Set<String> tags) {
|
||||
ApplicationReport report =
|
||||
newInstance(applicationId, applicationAttemptId, user, queue, name,
|
||||
host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
|
||||
finishTime, finalStatus, appResources, origTrackingUrl, progress,
|
||||
applicationType, amRmToken);
|
||||
report.setApplicationTags(tags);
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the <code>ApplicationId</code> of the application.
|
||||
* @return <code>ApplicationId</code> of the application
|
||||
|
|
|
@ -19,11 +19,14 @@
|
|||
package org.apache.hadoop.yarn.server.applicationhistoryservice;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
@ -249,6 +252,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
|
||||
YarnApplicationState state = YarnApplicationState.ACCEPTED;
|
||||
ApplicationResourceUsageReport appResources = null;
|
||||
Set<String> appTags = null;
|
||||
Map<ApplicationAccessType, String> appViewACLs =
|
||||
new HashMap<ApplicationAccessType, String>();
|
||||
Map<String, Object> entityInfo = entity.getOtherInfo();
|
||||
|
@ -270,7 +274,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
ConverterUtils.toApplicationId(entity.getEntityId()),
|
||||
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
|
||||
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
|
||||
null, progress, type, null), appViewACLs);
|
||||
null, progress, type, null, appTags), appViewACLs);
|
||||
}
|
||||
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
|
||||
queue =
|
||||
|
@ -295,6 +299,17 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
appResources=ApplicationResourceUsageReport
|
||||
.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
|
||||
}
|
||||
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
|
||||
appTags = new HashSet<String>();
|
||||
Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
|
||||
if (obj != null && obj instanceof Collection<?>) {
|
||||
for(Object o : (Collection<?>)obj) {
|
||||
if (o != null) {
|
||||
appTags.add(o.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
List<TimelineEvent> events = entity.getEvents();
|
||||
if (events != null) {
|
||||
|
@ -347,7 +362,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
ConverterUtils.toApplicationId(entity.getEntityId()),
|
||||
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
|
||||
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
|
||||
null, progress, type, null), appViewACLs);
|
||||
null, progress, type, null, appTags), appViewACLs);
|
||||
}
|
||||
|
||||
private static ApplicationAttemptReport convertToApplicationAttemptReport(
|
||||
|
|
|
@ -22,7 +22,9 @@ import java.security.PrivilegedExceptionAction;
|
|||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
|
||||
|
@ -185,6 +187,9 @@ public class TestApplicationHistoryManagerOnTimelineStore {
|
|||
Assert.assertEquals(Integer.MAX_VALUE + 3L
|
||||
+ +app.getApplicationId().getId(), app.getFinishTime());
|
||||
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
|
||||
Assert.assertEquals(2, app.getApplicationTags().size());
|
||||
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
|
||||
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
|
||||
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
|
||||
// Nobody except admin and owner has access to the details of the app.
|
||||
if ((i == 1 && callerUGI != null &&
|
||||
|
@ -471,6 +476,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
|
|||
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
|
||||
"user2");
|
||||
}
|
||||
Set<String> appTags = new HashSet<String>();
|
||||
appTags.add("Test_APP_TAGS_1");
|
||||
appTags.add("Test_APP_TAGS_2");
|
||||
entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, appTags);
|
||||
entity.setOtherInfo(entityInfo);
|
||||
TimelineEvent tEvent = new TimelineEvent();
|
||||
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
|
||||
|
|
|
@ -73,4 +73,5 @@ public class ApplicationMetricsConstants {
|
|||
public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
|
||||
"YARN_APPLICATION_LATEST_APP_ATTEMPT";
|
||||
|
||||
public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS";
|
||||
}
|
||||
|
|
|
@ -18,6 +18,8 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
|
||||
public class ApplicationCreatedEvent extends
|
||||
|
@ -29,6 +31,7 @@ public class ApplicationCreatedEvent extends
|
|||
private String user;
|
||||
private String queue;
|
||||
private long submittedTime;
|
||||
private Set<String> appTags;
|
||||
|
||||
public ApplicationCreatedEvent(ApplicationId appId,
|
||||
String name,
|
||||
|
@ -36,7 +39,8 @@ public class ApplicationCreatedEvent extends
|
|||
String user,
|
||||
String queue,
|
||||
long submittedTime,
|
||||
long createdTime) {
|
||||
long createdTime,
|
||||
Set<String> appTags) {
|
||||
super(SystemMetricsEventType.APP_CREATED, createdTime);
|
||||
this.appId = appId;
|
||||
this.name = name;
|
||||
|
@ -44,6 +48,7 @@ public class ApplicationCreatedEvent extends
|
|||
this.user = user;
|
||||
this.queue = queue;
|
||||
this.submittedTime = submittedTime;
|
||||
this.appTags = appTags;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -75,4 +80,7 @@ public class ApplicationCreatedEvent extends
|
|||
return submittedTime;
|
||||
}
|
||||
|
||||
public Set<String> getAppTags() {
|
||||
return appTags;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -106,7 +106,7 @@ public class SystemMetricsPublisher extends CompositeService {
|
|||
app.getUser(),
|
||||
app.getQueue(),
|
||||
app.getSubmitTime(),
|
||||
createdTime));
|
||||
createdTime, app.getApplicationTags()));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -251,6 +251,8 @@ public class SystemMetricsPublisher extends CompositeService {
|
|||
event.getQueue());
|
||||
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
|
||||
event.getSubmittedTime());
|
||||
entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
|
||||
event.getAppTags());
|
||||
entity.setOtherInfo(entityInfo);
|
||||
TimelineEvent tEvent = new TimelineEvent();
|
||||
tEvent.setEventType(
|
||||
|
|
|
@ -21,7 +21,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics;
|
|||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
|
@ -143,6 +147,8 @@ public class TestSystemMetricsPublisher {
|
|||
Assert.assertEquals(app.getSubmitTime(),
|
||||
entity.getOtherInfo().get(
|
||||
ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO));
|
||||
Assert.assertTrue(verifyAppTags(app.getApplicationTags(),
|
||||
entity.getOtherInfo()));
|
||||
if (i == 1) {
|
||||
Assert.assertEquals("uers1,user2",
|
||||
entity.getOtherInfo().get(
|
||||
|
@ -352,6 +358,10 @@ public class TestSystemMetricsPublisher {
|
|||
FinalApplicationStatus.UNDEFINED);
|
||||
when(app.getRMAppMetrics()).thenReturn(
|
||||
new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
|
||||
Set<String> appTags = new HashSet<String>();
|
||||
appTags.add("test");
|
||||
appTags.add("tags");
|
||||
when(app.getApplicationTags()).thenReturn(appTags);
|
||||
return app;
|
||||
}
|
||||
|
||||
|
@ -392,4 +402,31 @@ public class TestSystemMetricsPublisher {
|
|||
return container;
|
||||
}
|
||||
|
||||
private static boolean verifyAppTags(Set<String> appTags,
|
||||
Map<String, Object> entityInfo) {
|
||||
if (!entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
|
||||
return false;
|
||||
}
|
||||
Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
|
||||
if (obj instanceof Collection<?>) {
|
||||
Collection<?> collection = (Collection<?>) obj;
|
||||
if (collection.size() != appTags.size()) {
|
||||
return false;
|
||||
}
|
||||
for (String appTag : appTags) {
|
||||
boolean match = false;
|
||||
for (Object o : collection) {
|
||||
if (o.toString().equals(appTag)) {
|
||||
match = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!match) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1125,7 +1125,27 @@ Response Body:
|
|||
"submittedTime":1430424769395,
|
||||
"startedTime":1430424769395,
|
||||
"finishedTime":1430424776594,
|
||||
"elapsedTime":7199
|
||||
"elapsedTime":7199},
|
||||
{
|
||||
"appId":"application_1430424020775_0001",
|
||||
"currentAppAttemptId":"appattempt_1430424020775_0001_000001",
|
||||
"user":"zshen",
|
||||
"name":"QuasiMonteCarlo",
|
||||
"queue":"default",
|
||||
"type":"MAPREDUCE",
|
||||
"host":"localhost",
|
||||
"rpcPort":56264,
|
||||
"appState":"FINISHED",
|
||||
"progress":100.0,
|
||||
"diagnosticsInfo":"",
|
||||
"originalTrackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:19888/jobhistory/job/job_1430424020775_0001",
|
||||
"trackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:8088/proxy/application_1430424020775_0001/",
|
||||
"finalAppStatus":"SUCCEEDED",
|
||||
"submittedTime":1430424053809,
|
||||
"startedTime":1430424072153,
|
||||
"finishedTime":1430424776594,
|
||||
"elapsedTime":18344,
|
||||
"applicationTags":"mrapplication,ta-example"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
@ -1227,6 +1247,7 @@ Response Body:
|
|||
<startedTime>1430424053809</startedTime>
|
||||
<finishedTime>1430424072153</finishedTime>
|
||||
<elapsedTime>18344</elapsedTime>
|
||||
<applicationTags>mrapplication,ta-example</applicationTags>
|
||||
</app>
|
||||
</apps>
|
||||
|
||||
|
@ -1275,7 +1296,8 @@ None
|
|||
| `allocatedVCores` | int | The sum of virtual cores allocated to the application's running containers |
|
||||
| `currentAppAttemptId` | string | The latest application attempt ID |
|
||||
| `host` | string | The host of the ApplicationMaster |
|
||||
| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared. |
|
||||
| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared |
|
||||
| `applicationTags` | string | The application tags. |
|
||||
|
||||
### Response Examples:
|
||||
|
||||
|
@ -1311,7 +1333,8 @@ Response Body:
|
|||
"submittedTime": 1430424053809,
|
||||
"startedTime": 1430424053809,
|
||||
"finishedTime": 1430424072153,
|
||||
"elapsedTime": 18344
|
||||
"elapsedTime": 18344,
|
||||
"applicationTags": mrapplication,tag-example
|
||||
}
|
||||
|
||||
#### XML response
|
||||
|
@ -1349,6 +1372,7 @@ Response Body:
|
|||
<startedTime>1430424053809</startedTime>
|
||||
<finishedTime>1430424072153</finishedTime>
|
||||
<elapsedTime>18344</elapsedTime>
|
||||
<applicationTags>mrapplication,ta-example</applicationTags>
|
||||
</app>
|
||||
|
||||
## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List
|
||||
|
|
Loading…
Reference in New Issue