Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3360. Add JMX metrics to TimelineDataManager (Jason Lowe via jeagles)

This commit is contained in:
Jonathan Eagles 2015-06-24 16:16:52 -05:00
parent 2236b577a3
commit 4c659ddbf7
7 changed files with 303 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -60,6 +60,8 @@ Release 2.8.0 - UNRELEASED
NEW FEATURES
YARN-3360. Add JMX metrics to TimelineDataManager (Jason Lowe via jeagles)
YARN-3345. Add non-exclusive node label API. (Wangda Tan via jianhe)
YARN-3365. Enhanced NodeManager to support using the 'tc' tool via

123
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java
View File

@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
@ -56,6 +57,7 @@ public class TimelineDataManager extends AbstractService {
@VisibleForTesting
public static final String DEFAULT_DOMAIN_ID = "DEFAULT";
private TimelineDataManagerMetrics metrics;
private TimelineStore store;
private TimelineACLsManager timelineACLsManager;
@ -69,6 +71,7 @@ public class TimelineDataManager extends AbstractService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
metrics = TimelineDataManagerMetrics.create();
TimelineDomain domain = store.getDomain("DEFAULT");
// it is okay to reuse an existing domain even if it was created by another
// user of the timeline server before, because it allows everybody to access.
@ -130,6 +133,38 @@ public class TimelineDataManager extends AbstractService {
Long limit,
EnumSet<Field> fields,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetEntitiesOps();
try {
TimelineEntities entities = doGetEntities(
entityType,
primaryFilter,
secondaryFilter,
windowStart,
windowEnd,
fromId,
fromTs,
limit,
fields,
callerUGI);
metrics.incrGetEntitiesTotal(entities.getEntities().size());
return entities;
} finally {
metrics.addGetEntitiesTime(Time.monotonicNow() - startTime);
}
}
private TimelineEntities doGetEntities(
String entityType,
NameValuePair primaryFilter,
Collection<NameValuePair> secondaryFilter,
Long windowStart,
Long windowEnd,
String fromId,
Long fromTs,
Long limit,
EnumSet<Field> fields,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineEntities entities = null;
entities = store.getEntities(
entityType,
@ -161,6 +196,20 @@ public class TimelineDataManager extends AbstractService {
String entityId,
EnumSet<Field> fields,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetEntityOps();
try {
return doGetEntity(entityType, entityId, fields, callerUGI);
} finally {
metrics.addGetEntityTime(Time.monotonicNow() - startTime);
}
}
private TimelineEntity doGetEntity(
String entityType,
String entityId,
EnumSet<Field> fields,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineEntity entity = null;
entity =
store.getEntity(entityId, entityType, fields);
@ -190,6 +239,32 @@ public class TimelineDataManager extends AbstractService {
Long windowEnd,
Long limit,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetEventsOps();
try {
TimelineEvents events = doGetEvents(
entityType,
entityIds,
eventTypes,
windowStart,
windowEnd,
limit,
callerUGI);
metrics.incrGetEventsTotal(events.getAllEvents().size());
return events;
} finally {
metrics.addGetEventsTime(Time.monotonicNow() - startTime);
}
}
private TimelineEvents doGetEvents(
String entityType,
SortedSet<String> entityIds,
SortedSet<String> eventTypes,
Long windowStart,
Long windowEnd,
Long limit,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineEvents events = null;
events = store.getEntityTimelines(
entityType,
@ -236,9 +311,22 @@ public class TimelineDataManager extends AbstractService {
public TimelinePutResponse postEntities(
TimelineEntities entities,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrPostEntitiesOps();
try {
return doPostEntities(entities, callerUGI);
} finally {
metrics.addPostEntitiesTime(Time.monotonicNow() - startTime);
}
}
private TimelinePutResponse doPostEntities(
TimelineEntities entities,
UserGroupInformation callerUGI) throws YarnException, IOException {
if (entities == null) {
return new TimelinePutResponse();
}
metrics.incrPostEntitiesTotal(entities.getEntities().size());
TimelineEntities entitiesToPut = new TimelineEntities();
List<TimelinePutResponse.TimelinePutError> errors =
new ArrayList<TimelinePutResponse.TimelinePutError>();
@ -303,6 +391,17 @@ public class TimelineDataManager extends AbstractService {
*/
public void putDomain(TimelineDomain domain,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrPutDomainOps();
try {
doPutDomain(domain, callerUGI);
} finally {
metrics.addPutDomainTime(Time.monotonicNow() - startTime);
}
}
private void doPutDomain(TimelineDomain domain,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineDomain existingDomain =
store.getDomain(domain.getId());
if (existingDomain != null) {
@ -329,6 +428,17 @@ public class TimelineDataManager extends AbstractService {
*/
public TimelineDomain getDomain(String domainId,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetDomainOps();
try {
return doGetDomain(domainId, callerUGI);
} finally {
metrics.addGetDomainTime(Time.monotonicNow() - startTime);
}
}
private TimelineDomain doGetDomain(String domainId,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineDomain domain = store.getDomain(domainId);
if (domain != null) {
if (timelineACLsManager.checkAccess(callerUGI, domain)) {
@ -344,6 +454,19 @@ public class TimelineDataManager extends AbstractService {
*/
public TimelineDomains getDomains(String owner,
UserGroupInformation callerUGI) throws YarnException, IOException {
long startTime = Time.monotonicNow();
metrics.incrGetDomainsOps();
try {
TimelineDomains domains = doGetDomains(owner, callerUGI);
metrics.incrGetDomainsTotal(domains.getDomains().size());
return domains;
} finally {
metrics.addGetDomainsTime(Time.monotonicNow() - startTime);
}
}
private TimelineDomains doGetDomains(String owner,
UserGroupInformation callerUGI) throws YarnException, IOException {
TimelineDomains domains = store.getDomains(owner);
boolean hasAccess = true;
if (domains.getDomains().size() > 0) {

174
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManagerMetrics.java Normal file
View File

@ -0,0 +1,174 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.timeline;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableRate;
/** This class tracks metrics for the TimelineDataManager. */
@Metrics(about="Metrics for TimelineDataManager", context="yarn")
public class TimelineDataManagerMetrics {
@Metric("getEntities calls")
MutableCounterLong getEntitiesOps;
@Metric("Entities returned via getEntities")
MutableCounterLong getEntitiesTotal;
@Metric("getEntities processing time")
MutableRate getEntitiesTime;
@Metric("getEntity calls")
MutableCounterLong getEntityOps;
@Metric("getEntity processing time")
MutableRate getEntityTime;
@Metric("getEvents calls")
MutableCounterLong getEventsOps;
@Metric("Events returned via getEvents")
MutableCounterLong getEventsTotal;
@Metric("getEvents processing time")
MutableRate getEventsTime;
@Metric("postEntities calls")
MutableCounterLong postEntitiesOps;
@Metric("Entities posted via postEntities")
MutableCounterLong postEntitiesTotal;
@Metric("postEntities processing time")
MutableRate postEntitiesTime;
@Metric("putDomain calls")
MutableCounterLong putDomainOps;
@Metric("putDomain processing time")
MutableRate putDomainTime;
@Metric("getDomain calls")
MutableCounterLong getDomainOps;
@Metric("getDomain processing time")
MutableRate getDomainTime;
@Metric("getDomains calls")
MutableCounterLong getDomainsOps;
@Metric("Domains returned via getDomains")
MutableCounterLong getDomainsTotal;
@Metric("getDomains processing time")
MutableRate getDomainsTime;
@Metric("Total calls")
public long totalOps() {
return getEntitiesOps.value() +
getEntityOps.value() +
getEventsOps.value() +
postEntitiesOps.value() +
putDomainOps.value() +
getDomainOps.value() +
getDomainsOps.value();
}
TimelineDataManagerMetrics() {
}
public static TimelineDataManagerMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(new TimelineDataManagerMetrics());
}
public void incrGetEntitiesOps() {
getEntitiesOps.incr();
}
public void incrGetEntitiesTotal(long delta) {
getEntitiesTotal.incr(delta);
}
public void addGetEntitiesTime(long msec) {
getEntitiesTime.add(msec);
}
public void incrGetEntityOps() {
getEntityOps.incr();
}
public void addGetEntityTime(long msec) {
getEntityTime.add(msec);
}
public void incrGetEventsOps() {
getEventsOps.incr();
}
public void incrGetEventsTotal(long delta) {
getEventsTotal.incr(delta);
}
public void addGetEventsTime(long msec) {
getEventsTime.add(msec);
}
public void incrPostEntitiesOps() {
postEntitiesOps.incr();
}
public void incrPostEntitiesTotal(long delta) {
postEntitiesTotal.incr(delta);
}
public void addPostEntitiesTime(long msec) {
postEntitiesTime.add(msec);
}
public void incrPutDomainOps() {
putDomainOps.incr();
}
public void addPutDomainTime(long msec) {
putDomainTime.add(msec);
}
public void incrGetDomainOps() {
getDomainOps.incr();
}
public void addGetDomainTime(long msec) {
getDomainTime.add(msec);
}
public void incrGetDomainsOps() {
getDomainsOps.incr();
}
public void incrGetDomainsTotal(long delta) {
getDomainsTotal.incr(delta);
}
public void addGetDomainsTime(long msec) {
getDomainsTime.add(msec);
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
View File

@ -67,6 +67,7 @@ public class TestApplicationHistoryClientService {
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
dataManager =
new TimelineDataManager(store, aclsManager);
dataManager.init(conf);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -98,6 +98,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration());
TimelineDataManager dataManager =
new TimelineDataManager(store, aclsManager);
dataManager.init(conf);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
historyManager =
new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
View File

@ -94,6 +94,7 @@ public class TestAHSWebServices extends JerseyTestBase {
new TimelineDataManager(store, aclsManager);
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "foo");
dataManager.init(conf);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java
View File

@ -65,6 +65,7 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils {
dataManaer = new TimelineDataManager(store, aclsManager);
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
dataManaer.init(conf);
adminACLsManager = new AdminACLsManager(conf);
}