Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9559. Create AbstractContainersLauncher for pluggable ContainersLauncher logic. (Contributed by Jonathan Hung) 

(cherry picked from commit f51702d539)
This commit is contained in:
Haibo Chen 2019-08-06 13:51:11 -07:00 committed by Jonathan Hung
parent 3e8e1570f6
commit 8d357343c4
6 changed files with 119 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -3818,6 +3818,12 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD =
"/usr/bin/numactl";
/**
* Containers launcher implementation to use.
*/
public static final String NM_CONTAINERS_LAUNCHER_CLASS =
NM_PREFIX + "containers-launcher.class";
public YarnConfiguration() {
super();
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -4015,4 +4015,13 @@
<name>yarn.node-attribute.fs-store.impl.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore</value>
</property>
<property>
<description>
Containers launcher implementation for determining how containers
are launched within NodeManagers.
</description>
<name>yarn.nodemanager.containers-launcher.class</name>
<value>org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher</value>
</property>
</configuration>

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerTokenUpdatedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
@ -128,6 +129,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerReInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.AbstractContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.SignalContainersLauncherEvent;
@ -205,7 +207,7 @@ public class ContainerManagerImpl extends CompositeService implements
private final ContainersMonitor containersMonitor;
private Server server;
private final ResourceLocalizationService rsrcLocalizationSrvc;
private final ContainersLauncher containersLauncher;
private final AbstractContainersLauncher containersLauncher;
private final AuxServices auxiliaryServices;
private final NodeManagerMetrics metrics;
@ -571,9 +573,21 @@ public class ContainerManagerImpl extends CompositeService implements
return nmTimelinePublisherLocal;
}
protected ContainersLauncher createContainersLauncher(Context context,
ContainerExecutor exec) {
return new ContainersLauncher(context, this.dispatcher, exec, dirsHandler, this);
protected AbstractContainersLauncher createContainersLauncher(
Context ctxt, ContainerExecutor exec) {
Class<? extends AbstractContainersLauncher> containersLauncherClass =
ctxt.getConf()
.getClass(YarnConfiguration.NM_CONTAINERS_LAUNCHER_CLASS,
ContainersLauncher.class, AbstractContainersLauncher.class);
AbstractContainersLauncher launcher;
try {
launcher = ReflectionUtils.newInstance(containersLauncherClass,
ctxt.getConf());
launcher.init(ctxt, this.dispatcher, exec, dirsHandler, this);
} catch (Exception e) {
throw new RuntimeException(e);
}
return launcher;
}
protected EventHandler<ApplicationEvent> createApplicationEventDispatcher() {

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/AbstractContainersLauncher.java Normal file
View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
/**
* Pluggable ContainersLauncher interface for processing
* ContainersLauncherEvents.
*/
public interface AbstractContainersLauncher extends Service,
EventHandler<ContainersLauncherEvent> {
void init(Context context, Dispatcher dispatcher,
ContainerExecutor exec, LocalDirsHandlerService dirsHandler,
ContainerManagerImpl containerManager);
}

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
View File

@ -38,7 +38,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@ -58,15 +57,15 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*
*/
public class ContainersLauncher extends AbstractService
implements EventHandler<ContainersLauncherEvent> {
implements AbstractContainersLauncher {
private static final Logger LOG =
LoggerFactory.getLogger(ContainersLauncher.class);
private final Context context;
private final ContainerExecutor exec;
private final Dispatcher dispatcher;
private final ContainerManagerImpl containerManager;
private Context context;
private ContainerExecutor exec;
private Dispatcher dispatcher;
private ContainerManagerImpl containerManager;
private LocalDirsHandlerService dirsHandler;
@VisibleForTesting
@ -79,15 +78,27 @@ public class ContainersLauncher extends AbstractService
public final Map<ContainerId, ContainerLaunch> running =
Collections.synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
public ContainersLauncher() {
super("containers-launcher");
}
@VisibleForTesting
public ContainersLauncher(Context context, Dispatcher dispatcher,
ContainerExecutor exec, LocalDirsHandlerService dirsHandler,
ContainerManagerImpl containerManager) {
super("containers-launcher");
this.exec = exec;
this.context = context;
this.dispatcher = dispatcher;
this.dirsHandler = dirsHandler;
this.containerManager = containerManager;
this();
init(context, dispatcher, exec, dirsHandler, containerManager);
}
@Override
public void init(Context nmContext, Dispatcher nmDispatcher,
ContainerExecutor containerExec, LocalDirsHandlerService nmDirsHandler,
ContainerManagerImpl nmContainerManager) {
this.exec = containerExec;
this.context = nmContext;
this.dispatcher = nmDispatcher;
this.dirsHandler = nmDirsHandler;
this.containerManager = nmContainerManager;
}
@Override

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/package-info.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* This package contains classes related to NM container launch.
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;