YARN-9087. Improve logging for initialization of Resource plugins. Contributed by Szilard Nemeth.
This commit is contained in:
Sunil G
parent
bdde6a612e
commit
a95a0cbf2f
|
|
@ -312,11 +312,12 @@ public class LinuxContainerExecutor extends ContainerExecutor {
|
|||
resourceHandlerChain = ResourceHandlerModule
|
||||
.getConfiguredResourceHandlerChain(conf, nmContext);
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain
|
||||
!= null));
|
||||
final boolean enabled = resourceHandlerChain != null;
|
||||
LOG.debug("Resource handler chain enabled = " + enabled);
|
||||
}
|
||||
if (resourceHandlerChain != null) {
|
||||
LOG.debug("Bootstrapping resource handler chain");
|
||||
LOG.debug("Bootstrapping resource handler chain: " +
|
||||
resourceHandlerChain);
|
||||
resourceHandlerChain.bootstrap(conf);
|
||||
}
|
||||
} catch (ResourceHandlerException e) {
|
||||
|
|
|
|||
|
|
@ -173,4 +173,9 @@ public class CGroupsBlkioResourceHandlerImpl implements DiskResourceHandler {
|
|||
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return CGroupsBlkioResourceHandlerImpl.class.getName();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -256,4 +256,9 @@ public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
|
|||
throws ResourceHandlerException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return CGroupsCpuResourceHandlerImpl.class.getName();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
|
|||
private static final String MTAB_FILE = "/proc/mounts";
|
||||
private static final String CGROUPS_FSTYPE = "cgroup";
|
||||
|
||||
private String mtabFile;
|
||||
private final String mtabFile;
|
||||
private final String cGroupPrefix;
|
||||
private final boolean enableCGroupMount;
|
||||
private final String cGroupMountPath;
|
||||
|
|
@ -621,4 +621,16 @@ class CGroupsHandlerImpl implements CGroupsHandler {
|
|||
public String getCGroupMountPath() {
|
||||
return cGroupMountPath;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return CGroupsHandlerImpl.class.getName() + "{" +
|
||||
"mtabFile='" + mtabFile + '\'' +
|
||||
", cGroupPrefix='" + cGroupPrefix + '\'' +
|
||||
", enableCGroupMount=" + enableCGroupMount +
|
||||
", cGroupMountPath='" + cGroupMountPath + '\'' +
|
||||
", deleteCGroupTimeout=" + deleteCGroupTimeout +
|
||||
", deleteCGroupDelay=" + deleteCGroupDelay +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -196,4 +196,9 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
|
|||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return CGroupsMemoryResourceHandlerImpl.class.getName();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -166,4 +166,9 @@ public class NetworkPacketTaggingHandlerImpl
|
|||
Configuration conf) {
|
||||
return NetworkTagMappingManagerFactory.getManager(conf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return NetworkPacketTaggingHandlerImpl.class.getName();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -159,4 +159,10 @@ public class ResourceHandlerChain implements ResourceHandler {
|
|||
return Collections.unmodifiableList(resourceHandlers);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return ResourceHandlerChain.class.getName() + "{" +
|
||||
"resourceHandlers=" + resourceHandlers +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -84,6 +84,9 @@ public class ResourceHandlerModule {
|
|||
if (cGroupsHandler == null) {
|
||||
cGroupsHandler = new CGroupsHandlerImpl(conf,
|
||||
PrivilegedOperationExecutor.getInstance(conf));
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Value of CGroupsHandler is: " + cGroupsHandler);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -306,18 +309,34 @@ public class ResourceHandlerModule {
|
|||
List<ResourceHandler> handlerList, Configuration conf,
|
||||
Context nmContext) throws ResourceHandlerException {
|
||||
ResourcePluginManager pluginManager = nmContext.getResourcePluginManager();
|
||||
if (pluginManager != null) {
|
||||
|
||||
if (pluginManager == null) {
|
||||
LOG.warn("Plugin manager was null while trying to add " +
|
||||
"ResourceHandlers from configuration!");
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
|
||||
if (pluginMap != null) {
|
||||
if (pluginMap == null) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("List of plugins of ResourcePluginManager was empty " +
|
||||
"while trying to add ResourceHandlers from configuration!");
|
||||
}
|
||||
return;
|
||||
} else {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("List of plugins of ResourcePluginManager: " +
|
||||
pluginManager.getNameToPlugins());
|
||||
}
|
||||
}
|
||||
|
||||
for (ResourcePlugin plugin : pluginMap.values()) {
|
||||
addHandlerIfNotNull(handlerList, plugin
|
||||
.createResourceHandler(nmContext,
|
||||
addHandlerIfNotNull(handlerList,
|
||||
plugin.createResourceHandler(nmContext,
|
||||
getInitializedCGroupsHandler(conf),
|
||||
PrivilegedOperationExecutor.getInstance(conf)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static ResourceHandlerChain getConfiguredResourceHandlerChain(
|
||||
Configuration conf, Context nmContext) throws ResourceHandlerException {
|
||||
|
|
|
|||
|
|
@ -283,4 +283,9 @@ public class TrafficControlBandwidthHandlerImpl
|
|||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return TrafficControlBandwidthHandlerImpl.class.getName();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,11 +55,11 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
|
|||
|
||||
private final String REQUEST_FPGA_IP_ID_KEY = "REQUESTED_FPGA_IP_ID";
|
||||
|
||||
private AbstractFpgaVendorPlugin vendorPlugin;
|
||||
private final AbstractFpgaVendorPlugin vendorPlugin;
|
||||
|
||||
private FpgaResourceAllocator allocator;
|
||||
private final FpgaResourceAllocator allocator;
|
||||
|
||||
private CGroupsHandler cGroupsHandler;
|
||||
private final CGroupsHandler cGroupsHandler;
|
||||
|
||||
public static final String EXCLUDED_FPGAS_CLI_OPTION = "--excluded_fpgas";
|
||||
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
|
||||
|
|
@ -223,4 +223,12 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
|
|||
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return FpgaResourceHandlerImpl.class.getName() + "{" +
|
||||
"vendorPlugin=" + vendorPlugin +
|
||||
", allocator=" + allocator +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -301,4 +301,9 @@ public class GpuResourceAllocator {
|
|||
}
|
||||
return assigns;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return GpuResourceAllocator.class.getName();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,10 +48,10 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
|
|||
public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
|
||||
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
|
||||
|
||||
private Context nmContext;
|
||||
private GpuResourceAllocator gpuAllocator;
|
||||
private CGroupsHandler cGroupsHandler;
|
||||
private PrivilegedOperationExecutor privilegedOperationExecutor;
|
||||
private final Context nmContext;
|
||||
private final GpuResourceAllocator gpuAllocator;
|
||||
private final CGroupsHandler cGroupsHandler;
|
||||
private final PrivilegedOperationExecutor privilegedOperationExecutor;
|
||||
|
||||
public GpuResourceHandlerImpl(Context nmContext,
|
||||
CGroupsHandler cGroupsHandler,
|
||||
|
|
@ -184,4 +184,11 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
|
|||
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return GpuResourceHandlerImpl.class.getName() + "{" +
|
||||
"gpuAllocator=" + gpuAllocator +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,8 +41,8 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
|
|||
|
||||
private static final Log LOG = LogFactory
|
||||
.getLog(NumaResourceHandlerImpl.class);
|
||||
private NumaResourceAllocator numaResourceAllocator;
|
||||
private String numaCtlCmd;
|
||||
private final NumaResourceAllocator numaResourceAllocator;
|
||||
private final String numaCtlCmd;
|
||||
|
||||
public NumaResourceHandlerImpl(Configuration conf, Context nmContext) {
|
||||
LOG.info("NUMA resources allocation is enabled, initializing NUMA resources"
|
||||
|
|
@ -111,4 +111,12 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
|
|||
public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return NumaResourceHandlerImpl.class.getName() + "{" +
|
||||
"numaResourceAllocator=" + numaResourceAllocator +
|
||||
", numaCtlCmd='" + numaCtlCmd + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
|
@ -52,7 +53,13 @@ public class ResourcePluginManager {
|
|||
public synchronized void initialize(Context context)
|
||||
throws YarnException {
|
||||
Configuration conf = context.getConf();
|
||||
|
||||
String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
|
||||
if (plugins == null || plugins.length == 0) {
|
||||
LOG.info("No Resource plugins found from configuration!");
|
||||
}
|
||||
LOG.info("Found Resource plugins from configuration: "
|
||||
+ Arrays.toString(plugins));
|
||||
|
||||
if (plugins != null) {
|
||||
Map<String, ResourcePlugin> pluginMap = new HashMap<>();
|
||||
|
|
@ -64,23 +71,21 @@ public class ResourcePluginManager {
|
|||
String msg =
|
||||
"Trying to initialize resource plugin with name=" + resourceName
|
||||
+ ", it is not supported, list of supported plugins:"
|
||||
+ StringUtils.join(",",
|
||||
SUPPORTED_RESOURCE_PLUGINS);
|
||||
+ StringUtils.join(",", SUPPORTED_RESOURCE_PLUGINS);
|
||||
LOG.error(msg);
|
||||
throw new YarnException(msg);
|
||||
}
|
||||
|
||||
if (pluginMap.containsKey(resourceName)) {
|
||||
// Duplicated items, ignore ...
|
||||
LOG.warn("Ignoring duplicate Resource plugin definition: " +
|
||||
resourceName);
|
||||
continue;
|
||||
}
|
||||
|
||||
ResourcePlugin plugin = null;
|
||||
if (resourceName.equals(GPU_URI)) {
|
||||
plugin = new GpuResourcePlugin();
|
||||
}
|
||||
|
||||
if (resourceName.equals(FPGA_URI)) {
|
||||
} else if (resourceName.equals(FPGA_URI)) {
|
||||
plugin = new FpgaResourcePlugin();
|
||||
}
|
||||
|
||||
|
|
@ -90,6 +95,7 @@ public class ResourcePluginManager {
|
|||
+ " should be loaded and initialized");
|
||||
}
|
||||
plugin.initialize(context);
|
||||
LOG.info("Initialized plugin {}", plugin);
|
||||
pluginMap.put(resourceName, plugin);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -102,4 +102,9 @@ public class FpgaResourcePlugin implements ResourcePlugin {
|
|||
public NMResourceInfo getNMResourceInfo() throws YarnException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return FpgaResourcePlugin.class.getName();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,4 +89,9 @@ public class GpuResourcePlugin implements ResourcePlugin {
|
|||
return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus,
|
||||
assignedGpuDevices);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return GpuResourcePlugin.class.getName();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue