Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-11496. Improve TimelineService log format. (#5677). Contributed by Xianming Lei. 

Reviewed-by: Shilun Fan <slfan1989@apache.org>
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
Xianming Lei 2023-05-20 17:27:45 +08:00 committed by GitHub
parent 9a524ede87
commit 0110e24ed8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 94 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
View File

@ -181,7 +181,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
return tokenMgrService.renewToken(appCollector.getDelegationTokenForApp(), return tokenMgrService.renewToken(appCollector.getDelegationTokenForApp(),
appCollector.getAppDelegationTokenRenewer()); appCollector.getAppDelegationTokenRenewer());
} else { } else {
LOG.info("Delegation token not available for renewal for app " + LOG.info("Delegation token not available for renewal for app {}",
appCollector.getTimelineEntityContext().getAppId()); appCollector.getTimelineEntityContext().getAppId());
return -1; return -1;
} }
@ -226,7 +226,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
renewalOrRegenerationFuture, tokenId.getMaxDate(), renewalOrRegenerationFuture, tokenId.getMaxDate(),
tokenId.getRenewer().toString()); tokenId.getRenewer().toString());
} }
LOG.info("Generated a new token " + timelineToken + " for app " + appId); LOG.info("Generated a new token {} for app {}", timelineToken, appId);
return org.apache.hadoop.yarn.api.records.Token.newInstance( return org.apache.hadoop.yarn.api.records.Token.newInstance(
timelineToken.getIdentifier(), timelineToken.getKind().toString(), timelineToken.getIdentifier(), timelineToken.getKind().toString(),
timelineToken.getPassword(), timelineToken.getService().toString()); timelineToken.getPassword(), timelineToken.getService().toString());
@ -249,7 +249,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
reportNewCollectorInfoToNM(appId, token); reportNewCollectorInfoToNM(appId, token);
} catch (YarnException | IOException e) { } catch (YarnException | IOException e) {
// throw exception here as it cannot be used if failed communicate with NM // throw exception here as it cannot be used if failed communicate with NM
LOG.error("Failed to communicate with NM Collector Service for " + appId); LOG.error("Failed to communicate with NM Collector Service for {}", appId);
throw new YarnRuntimeException(e); throw new YarnRuntimeException(e);
} }
} }
@ -260,7 +260,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
try { try {
cancelTokenForAppCollector((AppLevelTimelineCollector) collector); cancelTokenForAppCollector((AppLevelTimelineCollector) collector);
} catch (IOException e) { } catch (IOException e) {
LOG.warn("Failed to cancel token for app collector with appId " + LOG.warn("Failed to cancel token for app collector with appId {}",
appId, e); appId, e);
} }
} }
@ -328,7 +328,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
//TODO: We need to think of the case of multiple interfaces //TODO: We need to think of the case of multiple interfaces
this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress( this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
timelineRestServer.getConnectorAddress(0)); timelineRestServer.getConnectorAddress(0));
LOG.info("Instantiated the per-node collector webapp at " + LOG.info("Instantiated the per-node collector webapp at {}",
timelineRestServerBindAddress); timelineRestServerBindAddress);
} }
@ -338,8 +338,8 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
ReportNewCollectorInfoRequest request = ReportNewCollectorInfoRequest request =
ReportNewCollectorInfoRequest.newInstance(appId, ReportNewCollectorInfoRequest.newInstance(appId,
this.timelineRestServerBindAddress, token); this.timelineRestServerBindAddress, token);
LOG.info("Report a new collector for application: " + appId + LOG.info("Report a new collector for application: {}" +
" to the NM Collector Service."); " to the NM Collector Service.", appId);
getNMCollectorService().reportNewCollectorInfo(request); getNMCollectorService().reportNewCollectorInfo(request);
} }
@ -348,7 +348,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
throws YarnException, IOException { throws YarnException, IOException {
GetTimelineCollectorContextRequest request = GetTimelineCollectorContextRequest request =
GetTimelineCollectorContextRequest.newInstance(appId); GetTimelineCollectorContextRequest.newInstance(appId);
LOG.info("Get timeline collector context for " + appId); LOG.info("Get timeline collector context for {}", appId);
GetTimelineCollectorContextResponse response = GetTimelineCollectorContextResponse response =
getNMCollectorService().getTimelineCollectorContext(request); getNMCollectorService().getTimelineCollectorContext(request);
String userId = response.getUserId(); String userId = response.getUserId();
@ -384,7 +384,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS, YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT); YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
LOG.info("nmCollectorServiceAddress: " + nmCollectorServiceAddress); LOG.info("nmCollectorServiceAddress: {}", nmCollectorServiceAddress);
final YarnRPC rpc = YarnRPC.create(conf); final YarnRPC rpc = YarnRPC.create(conf);
// TODO Security settings. // TODO Security settings.
@ -418,8 +418,8 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
// Set renewal or regeneration timer based on delay. // Set renewal or regeneration timer based on delay.
long renewalDelay = 0; long renewalDelay = 0;
if (newExpirationTime > 0) { if (newExpirationTime > 0) {
LOG.info("Renewed token for " + appId + " with new expiration " + LOG.info("Renewed token for {} with new expiration " +
"timestamp = " + newExpirationTime); "timestamp = {}", appId, newExpirationTime);
renewalDelay = getRenewalDelay(newExpirationTime - Time.now()); renewalDelay = getRenewalDelay(newExpirationTime - Time.now());
} }
long regenerationDelay = long regenerationDelay =
@ -442,7 +442,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
try { try {
reportNewCollectorInfoToNM(appId, token); reportNewCollectorInfoToNM(appId, token);
} catch (YarnException e) { } catch (YarnException e) {
LOG.warn("Unable to report regenerated token to NM for " + appId); LOG.warn("Unable to report regenerated token to NM for {}", appId);
} }
} }
@ -450,8 +450,8 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
public void run() { public void run() {
TimelineCollector collector = get(appId); TimelineCollector collector = get(appId);
if (collector == null) { if (collector == null) {
LOG.info("Cannot find active collector while " + (timerForRenewal ? LOG.info("Cannot find active collector while {} token for {}",
"renewing" : "regenerating") + " token for " + appId); (timerForRenewal ? "renewing" : "regenerating"), appId);
return; return;
} }
AppLevelTimelineCollector appCollector = AppLevelTimelineCollector appCollector =
@ -466,8 +466,8 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
regenerateToken(appCollector); regenerateToken(appCollector);
} }
} catch (Exception e) { } catch (Exception e) {
LOG.warn("Unable to " + (timerForRenewal ? "renew" : "regenerate") + LOG.warn("Unable to {} token for {}",
" token for " + appId, e); (timerForRenewal ? "renew" : "regenerate"), appId, e);
} }
} }
} }

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
View File

@ -193,8 +193,8 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
synchronized (appIdToContainerId) { synchronized (appIdToContainerId) {
Set<ContainerId> masterContainers = appIdToContainerId.get(appId); Set<ContainerId> masterContainers = appIdToContainerId.get(appId);
if (masterContainers == null) { if (masterContainers == null) {
LOG.info("Stop container for " + containerId LOG.info("Stop container for {}"
+ " is called before initializing container."); + " is called before initializing container.", containerId);
return; return;
} }
masterContainers.remove(containerId); masterContainers.remove(containerId);

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
View File

@ -75,7 +75,7 @@ public class TimelineCollectorManager extends CompositeService {
String timelineWriterClassName = conf.get( String timelineWriterClassName = conf.get(
YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WRITER_CLASS); YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WRITER_CLASS);
LOG.info("Using TimelineWriter: " + timelineWriterClassName); LOG.info("Using TimelineWriter: {}", timelineWriterClassName);
try { try {
Class<?> timelineWriterClazz = Class.forName(timelineWriterClassName); Class<?> timelineWriterClazz = Class.forName(timelineWriterClassName);
if (TimelineWriter.class.isAssignableFrom(timelineWriterClazz)) { if (TimelineWriter.class.isAssignableFrom(timelineWriterClazz)) {
@ -139,14 +139,14 @@ public class TimelineCollectorManager extends CompositeService {
collector.setWriter(writer); collector.setWriter(writer);
collector.start(); collector.start();
collectors.put(appId, collector); collectors.put(appId, collector);
LOG.info("the collector for " + appId + " was added"); LOG.info("the collector for {} was added", appId);
collectorInTable = collector; collectorInTable = collector;
postPut(appId, collectorInTable); postPut(appId, collectorInTable);
} catch (Exception e) { } catch (Exception e) {
throw new YarnRuntimeException(e); throw new YarnRuntimeException(e);
} }
} else { } else {
LOG.info("the collector for " + appId + " already exists!"); LOG.info("the collector for {} already exists!", appId);
} }
} }
return collectorInTable; return collectorInTable;
@ -182,14 +182,14 @@ public class TimelineCollectorManager extends CompositeService {
public boolean remove(ApplicationId appId) { public boolean remove(ApplicationId appId) {
TimelineCollector collector = collectors.remove(appId); TimelineCollector collector = collectors.remove(appId);
if (collector == null) { if (collector == null) {
LOG.error("the collector for " + appId + " does not exist!"); LOG.error("the collector for {} does not exist!", appId);
} else { } else {
synchronized (collector) { synchronized (collector) {
postRemove(appId, collector); postRemove(appId, collector);
// stop the service to do clean up // stop the service to do clean up
collector.stop(); collector.stop();
} }
LOG.info("The collector service for " + appId + " was removed"); LOG.info("The collector service for {} was removed", appId);
} }
return collector != null; return collector != null;
} }

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
View File

@ -175,7 +175,7 @@ public class TimelineCollectorWebService {
NodeTimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY); NodeTimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY);
TimelineCollector collector = collectorManager.get(appID); TimelineCollector collector = collectorManager.get(appID);
if (collector == null) { if (collector == null) {
LOG.error("Application: "+ appId + " is not found"); LOG.error("Application: {} is not found", appId);
throw new NotFoundException("Application: "+ appId + " is not found"); throw new NotFoundException("Application: "+ appId + " is not found");
} }
@ -244,7 +244,7 @@ public class TimelineCollectorWebService {
NodeTimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY); NodeTimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY);
TimelineCollector collector = collectorManager.get(appID); TimelineCollector collector = collectorManager.get(appID);
if (collector == null) { if (collector == null) {
LOG.error("Application: " + appId + " is not found"); LOG.error("Application: {} is not found", appId);
throw new NotFoundException("Application: " + appId + " is not found"); throw new NotFoundException("Application: " + appId + " is not found");
} }
@ -270,7 +270,7 @@ public class TimelineCollectorWebService {
return null; return null;
} }
} catch (IllegalFormatException e) { } catch (IllegalFormatException e) {
LOG.error("Invalid application ID: " + appId); LOG.error("Invalid application ID: {}", appId);
return null; return null;
} }
} }

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
View File

@ -102,7 +102,7 @@ public class TimelineReaderServer extends CompositeService {
String timelineReaderClassName = conf.get( String timelineReaderClassName = conf.get(
YarnConfiguration.TIMELINE_SERVICE_READER_CLASS, YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READER_CLASS); YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READER_CLASS);
LOG.info("Using store: " + timelineReaderClassName); LOG.info("Using store: {}", timelineReaderClassName);
try { try {
Class<?> timelineReaderClazz = Class.forName(timelineReaderClassName); Class<?> timelineReaderClazz = Class.forName(timelineReaderClassName);
if (TimelineReader.class.isAssignableFrom(timelineReaderClazz)) { if (TimelineReader.class.isAssignableFrom(timelineReaderClazz)) {
@ -192,7 +192,7 @@ public class TimelineReaderServer extends CompositeService {
String bindAddress = WebAppUtils String bindAddress = WebAppUtils
.getWebAppBindURL(conf, hostProperty, webAppURLWithoutScheme); .getWebAppBindURL(conf, hostProperty, webAppURLWithoutScheme);
LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress); LOG.info("Instantiating TimelineReaderWebApp at {}", bindAddress);
try { try {
String httpScheme = WebAppUtils.getHttpSchemePrefix(conf); String httpScheme = WebAppUtils.getHttpSchemePrefix(conf);

133
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
View File

@ -179,8 +179,8 @@ public class TimelineReaderWebServices {
String invalidNumMsg) throws BadRequestException, String invalidNumMsg) throws BadRequestException,
WebApplicationException { WebApplicationException {
long endTime = Time.monotonicNow(); long endTime = Time.monotonicNow();
LOG.info("Processed URL " + url + " but encountered exception (Took " + LOG.info("Processed URL {} but encountered exception (Took " +
(endTime - startTime) + " ms.)"); "{} ms.)", url, (endTime - startTime));
if (e instanceof NumberFormatException) { if (e instanceof NumberFormatException) {
throw new BadRequestException(invalidNumMsg + " is not a numeric value."); throw new BadRequestException(invalidNumMsg + " is not a numeric value.");
} else if (e instanceof IllegalArgumentException) { } else if (e instanceof IllegalArgumentException) {
@ -356,8 +356,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -389,8 +389,8 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {}" +
" (Took " + latency + " ms.)"); " (Took {} ms.)", url, latency);
} }
if (entities == null) { if (entities == null) {
entities = Collections.emptySet(); entities = Collections.emptySet();
@ -659,8 +659,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -689,8 +689,7 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entities == null) { if (entities == null) {
entities = Collections.emptySet(); entities = Collections.emptySet();
@ -759,8 +758,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -784,12 +783,11 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entity == null) { if (entity == null) {
LOG.info("Processed URL " + url + " but entity not found" + " (Took " + LOG.info("Processed URL {} but entity not found" + " (Took {} ms.)",
(Time.monotonicNow() - startTime) + " ms.)"); url, (Time.monotonicNow() - startTime));
throw new NotFoundException("Timeline entity with uid: " + uId + throw new NotFoundException("Timeline entity with uid: " + uId +
"is not found"); "is not found");
} }
@ -808,8 +806,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -832,12 +830,11 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entity == null) { if (entity == null) {
LOG.info("Processed URL " + url + " but entity not found" + " (Took " + LOG.info("Processed URL {} but entity not found" + " (Took " +
(Time.monotonicNow() - startTime) + " ms.)"); "{} ms.)", url, (Time.monotonicNow() - startTime));
throw new NotFoundException("Timeline entity {id: " + entityId + throw new NotFoundException("Timeline entity {id: " + entityId +
", type: " + entityType + " } is not found"); ", type: " + entityType + " } is not found");
} }
@ -1040,8 +1037,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -1065,12 +1062,11 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entity == null) { if (entity == null) {
LOG.info("Processed URL " + url + " but flowrun not found (Took " + LOG.info("Processed URL {} but flowrun not found (Took {} ms.)",
(Time.monotonicNow() - startTime) + " ms.)"); url, (Time.monotonicNow() - startTime));
throw new NotFoundException("Flowrun with uid: " + uId + "is not found"); throw new NotFoundException("Flowrun with uid: " + uId + "is not found");
} }
return entity; return entity;
@ -1158,8 +1154,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -1182,12 +1178,11 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entity == null) { if (entity == null) {
LOG.info("Processed URL " + url + " but flowrun not found (Took " + LOG.info("Processed URL {} but flowrun not found (Took {} ms.)",
(Time.monotonicNow() - startTime) + " ms.)"); url, (Time.monotonicNow() - startTime));
throw new NotFoundException("Flow run {flow name: " + throw new NotFoundException("Flow run {flow name: " +
TimelineReaderWebServicesUtils.parseStr(flowName) + ", run id: " + TimelineReaderWebServicesUtils.parseStr(flowName) + ", run id: " +
TimelineReaderWebServicesUtils.parseLongStr(flowRunId) + TimelineReaderWebServicesUtils.parseLongStr(flowRunId) +
@ -1254,8 +1249,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -1283,8 +1278,7 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entities == null) { if (entities == null) {
entities = Collections.emptySet(); entities = Collections.emptySet();
@ -1413,8 +1407,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -1443,8 +1437,7 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entities == null) { if (entities == null) {
entities = Collections.emptySet(); entities = Collections.emptySet();
@ -1556,8 +1549,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -1581,8 +1574,7 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entities == null) { if (entities == null) {
entities = Collections.emptySet(); entities = Collections.emptySet();
@ -1654,8 +1646,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -1680,12 +1672,11 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entity == null) { if (entity == null) {
LOG.info("Processed URL " + url + " but app not found" + " (Took " + LOG.info("Processed URL {} but app not found" + " (Took " +
(Time.monotonicNow() - startTime) + " ms.)"); "{} ms.)", url, (Time.monotonicNow() - startTime));
throw new NotFoundException("App with uid " + uId + " not found"); throw new NotFoundException("App with uid " + uId + " not found");
} }
return entity; return entity;
@ -1833,8 +1824,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -1856,12 +1847,11 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entity == null) { if (entity == null) {
LOG.info("Processed URL " + url + " but app not found" + " (Took " + LOG.info("Processed URL {} but app not found" + " (Took " +
(Time.monotonicNow() - startTime) + " ms.)"); "{} ms.)", url, (Time.monotonicNow() - startTime));
throw new NotFoundException("App " + appId + " not found"); throw new NotFoundException("App " + appId + " not found");
} }
return entity; return entity;
@ -1971,8 +1961,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -2003,8 +1993,7 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entities == null) { if (entities == null) {
entities = Collections.emptySet(); entities = Collections.emptySet();
@ -3382,8 +3371,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -3401,8 +3390,7 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntityTypesLatency(latency, succeeded); METRICS.addGetEntityTypesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
return results; return results;
} }
@ -3468,8 +3456,8 @@ public class TimelineReaderWebServices {
QUERY_STRING_SEP + req.getQueryString()); QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " + LOG.info("Received URL {} from user {}",
TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -3496,8 +3484,7 @@ public class TimelineReaderWebServices {
} finally { } finally {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url + LOG.info("Processed URL {} (Took {} ms.)", url, latency);
" (Took " + latency + " ms.)");
} }
if (entities == null) { if (entities == null) {
entities = Collections.emptySet(); entities = Collections.emptySet();
@ -3544,8 +3531,8 @@ public class TimelineReaderWebServices {
: QUERY_STRING_SEP + req.getQueryString()); : QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI = UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req); TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " LOG.info("Received URL {} from user {}",
+ TimelineReaderWebServicesUtils.getUserName(callerUGI)); url, TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow(); long startTime = Time.monotonicNow();
boolean succeeded = false; boolean succeeded = false;
init(res); init(res);
@ -3569,7 +3556,7 @@ public class TimelineReaderWebServices {
long latency = Time.monotonicNow() - startTime; long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded); METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info( LOG.info(
"Processed URL " + url + " (Took " + latency + " ms.)"); "Processed URL {} (Took {} ms.)", url, latency);
} }
if (entities == null) { if (entities == null) {
entities = Collections.emptySet(); entities = Collections.emptySet();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
View File

@ -105,9 +105,9 @@ public class TimelineReaderWhitelistAuthorizationFilter implements Filter {
listAllowedUsers = listAllowedUsers =
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS; YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS;
} }
LOG.info("listAllowedUsers=" + listAllowedUsers); LOG.info("listAllowedUsers={}", listAllowedUsers);
allowedUsersAclList = new AccessControlList(listAllowedUsers); allowedUsersAclList = new AccessControlList(listAllowedUsers);
LOG.info("allowedUsersAclList=" + allowedUsersAclList.getUsers()); LOG.info("allowedUsersAclList={}", allowedUsersAclList.getUsers());
// also allow admins // also allow admins
String adminAclListStr = String adminAclListStr =
conf.getInitParameter(YarnConfiguration.YARN_ADMIN_ACL); conf.getInitParameter(YarnConfiguration.YARN_ADMIN_ACL);
@ -117,7 +117,7 @@ public class TimelineReaderWhitelistAuthorizationFilter implements Filter {
LOG.info("adminAclList not set, hence setting it to \"\""); LOG.info("adminAclList not set, hence setting it to \"\"");
} }
adminAclList = new AccessControlList(adminAclListStr); adminAclList = new AccessControlList(adminAclListStr);
LOG.info("adminAclList=" + adminAclList.getUsers()); LOG.info("adminAclList={}", adminAclList.getUsers());
} }
} }
} }

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
View File

@ -142,7 +142,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
.append("\n").toString().getBytes("UTF-8"); .append("\n").toString().getBytes("UTF-8");
writeFileWithRetries(filePath, record); writeFileWithRetries(filePath, record);
} catch (Exception ioe) { } catch (Exception ioe) {
LOG.warn("Interrupted operation:" + ioe.getMessage()); LOG.warn("Interrupted operation:{}", ioe.getMessage());
TimelineWriteError error = createTimelineWriteError(entity); TimelineWriteError error = createTimelineWriteError(entity);
/* /*
* TODO: set an appropriate error code after PoC could possibly be: * TODO: set an appropriate error code after PoC could possibly be:
@ -274,8 +274,8 @@ public class FileSystemTimelineWriterImpl extends AbstractService
LOG.info("Maxed out FS retries. Giving up!"); LOG.info("Maxed out FS retries. Giving up!");
throw e; throw e;
} }
LOG.info("Will retry operation on FS. Retry no. " + retry + LOG.info("Will retry operation on FS. Retry no. {}" +
" after sleeping for " + fsRetryInterval + " seconds"); " after sleeping for {} seconds", retry, fsRetryInterval);
Thread.sleep(fsRetryInterval); Thread.sleep(fsRetryInterval);
} }
} }