Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9087. Improve logging for initialization of Resource plugins. Contributed by Szilard Nemeth.

This commit is contained in:
Sunil G 2019-02-27 11:54:36 +05:30
parent bdde6a612e
commit a95a0cbf2f
16 changed files with 135 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
View File

@ -312,11 +312,12 @@ public class LinuxContainerExecutor extends ContainerExecutor {
resourceHandlerChain = ResourceHandlerModule
.getConfiguredResourceHandlerChain(conf, nmContext);
if (LOG.isDebugEnabled()) {
LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain
!= null));
final boolean enabled = resourceHandlerChain != null;
LOG.debug("Resource handler chain enabled = " + enabled);
}
if (resourceHandlerChain != null) {
LOG.debug("Bootstrapping resource handler chain");
LOG.debug("Bootstrapping resource handler chain: " +
resourceHandlerChain);
resourceHandlerChain.bootstrap(conf);
}
} catch (ResourceHandlerException e) {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
View File

@ -173,4 +173,9 @@ public class CGroupsBlkioResourceHandlerImpl implements DiskResourceHandler {
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return CGroupsBlkioResourceHandlerImpl.class.getName();
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
View File

@ -256,4 +256,9 @@ public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return CGroupsCpuResourceHandlerImpl.class.getName();
}
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
View File

@ -64,7 +64,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
private static final String MTAB_FILE = "/proc/mounts";
private static final String CGROUPS_FSTYPE = "cgroup";
private String mtabFile;
private final String mtabFile;
private final String cGroupPrefix;
private final boolean enableCGroupMount;
private final String cGroupMountPath;
@ -621,4 +621,16 @@ class CGroupsHandlerImpl implements CGroupsHandler {
public String getCGroupMountPath() {
return cGroupMountPath;
}
@Override
public String toString() {
return CGroupsHandlerImpl.class.getName() + "{" +
"mtabFile='" + mtabFile + '\'' +
", cGroupPrefix='" + cGroupPrefix + '\'' +
", enableCGroupMount=" + enableCGroupMount +
", cGroupMountPath='" + cGroupMountPath + '\'' +
", deleteCGroupTimeout=" + deleteCGroupTimeout +
", deleteCGroupDelay=" + deleteCGroupDelay +
'}';
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
View File

@ -196,4 +196,9 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
}
return Optional.empty();
}
@Override
public String toString() {
return CGroupsMemoryResourceHandlerImpl.class.getName();
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java
View File

@ -166,4 +166,9 @@ public class NetworkPacketTaggingHandlerImpl
Configuration conf) {
return NetworkTagMappingManagerFactory.getManager(conf);
}
@Override
public String toString() {
return NetworkPacketTaggingHandlerImpl.class.getName();
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java
View File

@ -159,4 +159,10 @@ public class ResourceHandlerChain implements ResourceHandler {
return Collections.unmodifiableList(resourceHandlers);
}
@Override
public String toString() {
return ResourceHandlerChain.class.getName() + "{" +
"resourceHandlers=" + resourceHandlers +
'}';
}
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
View File

@ -84,6 +84,9 @@ public class ResourceHandlerModule {
if (cGroupsHandler == null) {
cGroupsHandler = new CGroupsHandlerImpl(conf,
PrivilegedOperationExecutor.getInstance(conf));
if (LOG.isDebugEnabled()) {
LOG.debug("Value of CGroupsHandler is: " + cGroupsHandler);
}
}
}
}
@ -306,16 +309,32 @@ public class ResourceHandlerModule {
List<ResourceHandler> handlerList, Configuration conf,
Context nmContext) throws ResourceHandlerException {
ResourcePluginManager pluginManager = nmContext.getResourcePluginManager();
if (pluginManager != null) {
Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
if (pluginMap != null) {
for (ResourcePlugin plugin : pluginMap.values()) {
addHandlerIfNotNull(handlerList, plugin
.createResourceHandler(nmContext,
getInitializedCGroupsHandler(conf),
PrivilegedOperationExecutor.getInstance(conf)));
}
if (pluginManager == null) {
LOG.warn("Plugin manager was null while trying to add " +
"ResourceHandlers from configuration!");
return;
}
Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
if (pluginMap == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("List of plugins of ResourcePluginManager was empty " +
"while trying to add ResourceHandlers from configuration!");
}
return;
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("List of plugins of ResourcePluginManager: " +
pluginManager.getNameToPlugins());
}
}
for (ResourcePlugin plugin : pluginMap.values()) {
addHandlerIfNotNull(handlerList,
plugin.createResourceHandler(nmContext,
getInitializedCGroupsHandler(conf),
PrivilegedOperationExecutor.getInstance(conf)));
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
View File

@ -283,4 +283,9 @@ public class TrafficControlBandwidthHandlerImpl
return null;
}
@Override
public String toString() {
return TrafficControlBandwidthHandlerImpl.class.getName();
}
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
View File

@ -55,11 +55,11 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
private final String REQUEST_FPGA_IP_ID_KEY = "REQUESTED_FPGA_IP_ID";
private AbstractFpgaVendorPlugin vendorPlugin;
private final AbstractFpgaVendorPlugin vendorPlugin;
private FpgaResourceAllocator allocator;
private final FpgaResourceAllocator allocator;
private CGroupsHandler cGroupsHandler;
private final CGroupsHandler cGroupsHandler;
public static final String EXCLUDED_FPGAS_CLI_OPTION = "--excluded_fpgas";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
@ -223,4 +223,12 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return FpgaResourceHandlerImpl.class.getName() + "{" +
"vendorPlugin=" + vendorPlugin +
", allocator=" + allocator +
'}';
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
View File

@ -301,4 +301,9 @@ public class GpuResourceAllocator {
}
return assigns;
}
@Override
public String toString() {
return GpuResourceAllocator.class.getName();
}
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
View File

@ -48,10 +48,10 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
private Context nmContext;
private GpuResourceAllocator gpuAllocator;
private CGroupsHandler cGroupsHandler;
private PrivilegedOperationExecutor privilegedOperationExecutor;
private final Context nmContext;
private final GpuResourceAllocator gpuAllocator;
private final CGroupsHandler cGroupsHandler;
private final PrivilegedOperationExecutor privilegedOperationExecutor;
public GpuResourceHandlerImpl(Context nmContext,
CGroupsHandler cGroupsHandler,
@ -184,4 +184,11 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return GpuResourceHandlerImpl.class.getName() + "{" +
"gpuAllocator=" + gpuAllocator +
'}';
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java
View File

@ -41,8 +41,8 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
private static final Log LOG = LogFactory
.getLog(NumaResourceHandlerImpl.class);
private NumaResourceAllocator numaResourceAllocator;
private String numaCtlCmd;
private final NumaResourceAllocator numaResourceAllocator;
private final String numaCtlCmd;
public NumaResourceHandlerImpl(Configuration conf, Context nmContext) {
LOG.info("NUMA resources allocation is enabled, initializing NUMA resources"
@ -111,4 +111,12 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
return null;
}
@Override
public String toString() {
return NumaResourceHandlerImpl.class.getName() + "{" +
"numaResourceAllocator=" + numaResourceAllocator +
", numaCtlCmd='" + numaCtlCmd + '\'' +
'}';
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
View File

@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@ -52,7 +53,13 @@ public class ResourcePluginManager {
public synchronized void initialize(Context context)
throws YarnException {
Configuration conf = context.getConf();
String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
if (plugins == null || plugins.length == 0) {
LOG.info("No Resource plugins found from configuration!");
}
LOG.info("Found Resource plugins from configuration: "
+ Arrays.toString(plugins));
if (plugins != null) {
Map<String, ResourcePlugin> pluginMap = new HashMap<>();
@ -64,23 +71,21 @@ public class ResourcePluginManager {
String msg =
"Trying to initialize resource plugin with name=" + resourceName
+ ", it is not supported, list of supported plugins:"
+ StringUtils.join(",",
SUPPORTED_RESOURCE_PLUGINS);
+ StringUtils.join(",", SUPPORTED_RESOURCE_PLUGINS);
LOG.error(msg);
throw new YarnException(msg);
}
if (pluginMap.containsKey(resourceName)) {
// Duplicated items, ignore ...
LOG.warn("Ignoring duplicate Resource plugin definition: " +
resourceName);
continue;
}
ResourcePlugin plugin = null;
if (resourceName.equals(GPU_URI)) {
plugin = new GpuResourcePlugin();
}
if (resourceName.equals(FPGA_URI)) {
} else if (resourceName.equals(FPGA_URI)) {
plugin = new FpgaResourcePlugin();
}
@ -90,6 +95,7 @@ public class ResourcePluginManager {
+ " should be loaded and initialized");
}
plugin.initialize(context);
LOG.info("Initialized plugin {}", plugin);
pluginMap.put(resourceName, plugin);
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaResourcePlugin.java
View File

@ -102,4 +102,9 @@ public class FpgaResourcePlugin implements ResourcePlugin {
public NMResourceInfo getNMResourceInfo() throws YarnException {
return null;
}
@Override
public String toString() {
return FpgaResourcePlugin.class.getName();
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
View File

@ -89,4 +89,9 @@ public class GpuResourcePlugin implements ResourcePlugin {
return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus,
assignedGpuDevices);
}
@Override
public String toString() {
return GpuResourcePlugin.class.getName();
}
}