diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index e79ab47afed..8eb89dc86c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -153,6 +153,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements private NMNodeLabelsHandler nodeLabelsHandler; private final NodeLabelsProvider nodeLabelsProvider; + private boolean timelineServiceV2Enabled; public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) { @@ -240,6 +241,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements this.logAggregationEnabled = conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } @Override @@ -1177,7 +1180,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements newResource.toString()); } } - if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) { + if (timelineServiceV2Enabled) { updateTimelineCollectorData(response); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 41b74186cd6..3c63ec6c153 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -227,6 +227,7 @@ public class ContainerManagerImpl extends CompositeService implements // NM metrics publisher is set only if the timeline service v.2 is enabled private NMTimelinePublisher nmMetricsPublisher; + private boolean timelineServiceV2Enabled; public ContainerManagerImpl(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, @@ -263,11 +264,13 @@ public class ContainerManagerImpl extends CompositeService implements // initialize the metrics publisher if the timeline service v.2 is enabled // and the system publisher is enabled Configuration conf = context.getConf(); - if (YarnConfiguration.timelineServiceV2Enabled(conf) && - YarnConfiguration.systemMetricsPublisherEnabled(conf)) { - LOG.info("YARN system metrics publishing service is enabled"); - nmMetricsPublisher = createNMTimelinePublisher(context); - context.setNMTimelinePublisher(nmMetricsPublisher); + if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { + LOG.info("YARN system metrics publishing service is enabled"); + nmMetricsPublisher = createNMTimelinePublisher(context); + context.setNMTimelinePublisher(nmMetricsPublisher); + } + this.timelineServiceV2Enabled = true; } this.containersMonitor = createContainersMonitor(exec); addService(this.containersMonitor); @@ -1184,7 +1187,7 @@ public class ContainerManagerImpl extends CompositeService implements private FlowContext getFlowContext(ContainerLaunchContext launchContext, ApplicationId applicationID) { FlowContext flowContext = null; - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { String flowName = launchContext.getEnvironment() .get(TimelineUtils.FLOW_NAME_TAG_PREFIX); String flowVersion = launchContext.getEnvironment() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index ae2887966d9..49bc21c3363 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -96,6 +96,7 @@ public class ApplicationMasterService extends AbstractService implements new ConcurrentHashMap(); protected final RMContext rmContext; private final AMSProcessingChain amsProcessingChain; + private boolean timelineServiceV2Enabled; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { @@ -213,6 +214,8 @@ public class ApplicationMasterService extends AbstractService implements YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); super.serviceStart(); } @@ -303,7 +306,7 @@ public class ApplicationMasterService extends AbstractService implements rmContext.getRMApps().get(applicationAttemptId.getApplicationId()); // Remove collector address when app get finished. - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { ((RMAppImpl) rmApp).removeCollectorData(); } // checking whether the app exits in RMStateStore at first not to throw diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 2ae10310c77..379251e2294 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -222,6 +222,7 @@ public class ClientRMService extends AbstractService implements RMAppState.ACCEPTED, RMAppState.RUNNING); private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -292,6 +293,8 @@ public class ClientRMService extends AbstractService implements YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); super.serviceStart(); } @@ -571,7 +574,7 @@ public class ClientRMService extends AbstractService implements throw RPCUtil.getRemoteException(ie); } - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { // Sanity check for flow run String value = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 5168bd3cf22..d3ce2419b9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -107,12 +107,15 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { private RMContext rmContext; private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; @Override public void init(ApplicationMasterServiceContext amsContext, ApplicationMasterServiceProcessor nextProcessor) { this.rmContext = (RMContext)amsContext; this.resourceProfilesManager = rmContext.getResourceProfilesManager(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(rmContext.getYarnConfiguration()); } @Override @@ -318,8 +321,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { response.setNumClusterNodes(getScheduler().getNumClusterNodes()); // add collector address for this application - if (YarnConfiguration.timelineServiceV2Enabled( - getRmContext().getYarnConfiguration())) { + if (timelineServiceV2Enabled) { CollectorInfo collectorInfo = app.getCollectorInfo(); if (collectorInfo != null) { response.setCollectorInfo(collectorInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 9369eb2e2f0..7f9f51cbec0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -94,6 +94,7 @@ public class RMAppManager implements EventHandler, private final ApplicationACLsManager applicationACLsManager; private Configuration conf; private YarnAuthorizationProvider authorizer; + private boolean timelineServiceV2Enabled; public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, @@ -114,6 +115,8 @@ public class RMAppManager implements EventHandler, this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } this.authorizer = YarnAuthorizationProvider.getInstance(conf); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } /** @@ -451,7 +454,7 @@ public class RMAppManager implements EventHandler, throw new YarnException(message); } - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { // Start timeline collector for the submitted app application.startTimelineCollector(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index e9971928117..81be62814b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -124,6 +124,7 @@ public class ResourceTrackerService extends AbstractService implements private DynamicResourceConfiguration drConf; private final AtomicLong timelineCollectorVersion = new AtomicLong(0); + private boolean timelineServiceV2Enabled; public ResourceTrackerService(RMContext rmContext, NodesListManager nodesListManager, @@ -170,6 +171,8 @@ public class ResourceTrackerService extends AbstractService implements minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); if (YarnConfiguration.areNodeLabelsEnabled(conf)) { isDistributedNodeLabelsConf = @@ -578,9 +581,7 @@ public class ResourceTrackerService extends AbstractService implements NodeAction.SHUTDOWN, message); } - boolean timelineV2Enabled = - YarnConfiguration.timelineServiceV2Enabled(getConfig()); - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Check & update collectors info from request. updateAppCollectorsMap(request); } @@ -600,7 +601,7 @@ public class ResourceTrackerService extends AbstractService implements nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials); } - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Return collectors' map that NM needs to know setAppCollectorsMapToResponse(rmNode.getRunningApps(), nodeHeartBeatResponse); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 0bedb528dc4..3ec329f55b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -81,6 +81,7 @@ public class AMLauncher implements Runnable { private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; + private boolean timelineServiceV2Enabled; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -93,6 +94,8 @@ public class AMLauncher implements Runnable { this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } private void connect() throws IOException { @@ -239,7 +242,7 @@ public class AMLauncher implements Runnable { } private void setFlowContext(ContainerLaunchContext container) { - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { Map environment = container.getEnvironment(); ApplicationId applicationId = application.getAppAttemptId().getApplicationId();