YARN-9087. Improve logging for initialization of Resource plugins. Contributed by Szilard Nemeth.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -312,11 +312,12 @@ public class LinuxContainerExecutor extends ContainerExecutor {
       resourceHandlerChain = ResourceHandlerModule
           .getConfiguredResourceHandlerChain(conf, nmContext);
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain
+        final boolean enabled = resourceHandlerChain != null;
-            != null));
+        LOG.debug("Resource handler chain enabled = " + enabled);
       }
       if (resourceHandlerChain != null) {
-        LOG.debug("Bootstrapping resource handler chain");
+        LOG.debug("Bootstrapping resource handler chain: " +
+            resourceHandlerChain);
         resourceHandlerChain.bootstrap(conf);
       }
     } catch (ResourceHandlerException e) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java

@@ -167,4 +167,9 @@ public class CGroupsBlkioResourceHandlerImpl implements DiskResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return CGroupsBlkioResourceHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java

@@ -243,4 +243,9 @@ public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
       throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return CGroupsCpuResourceHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java

@@ -64,7 +64,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
   private static final String MTAB_FILE = "/proc/mounts";
   private static final String CGROUPS_FSTYPE = "cgroup";
 
-  private String mtabFile;
+  private final String mtabFile;
   private final String cGroupPrefix;
   private final boolean enableCGroupMount;
   private final String cGroupMountPath;
@@ -617,4 +617,16 @@ class CGroupsHandlerImpl implements CGroupsHandler {
   public String getCGroupMountPath() {
     return cGroupMountPath;
   }
+
+  @Override
+  public String toString() {
+    return CGroupsHandlerImpl.class.getName() + "{" +
+        "mtabFile='" + mtabFile + '\'' +
+        ", cGroupPrefix='" + cGroupPrefix + '\'' +
+        ", enableCGroupMount=" + enableCGroupMount +
+        ", cGroupMountPath='" + cGroupMountPath + '\'' +
+        ", deleteCGroupTimeout=" + deleteCGroupTimeout +
+        ", deleteCGroupDelay=" + deleteCGroupDelay +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java

@@ -176,4 +176,8 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
     return null;
   }
 
+  @Override
+  public String toString() {
+    return CGroupsMemoryResourceHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkPacketTaggingHandlerImpl.java

@@ -160,4 +160,9 @@ public class NetworkPacketTaggingHandlerImpl
       Configuration conf) {
     return NetworkTagMappingManagerFactory.getManager(conf);
   }
+
+  @Override
+  public String toString() {
+    return NetworkPacketTaggingHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerChain.java

@@ -141,4 +141,10 @@ public class ResourceHandlerChain implements ResourceHandler {
     return Collections.unmodifiableList(resourceHandlers);
   }
 
+  @Override
+  public String toString() {
+    return ResourceHandlerChain.class.getName() + "{" +
+        "resourceHandlers=" + resourceHandlers +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java

@@ -84,6 +84,9 @@ public class ResourceHandlerModule {
         if (cGroupsHandler == null) {
           cGroupsHandler = new CGroupsHandlerImpl(conf,
               PrivilegedOperationExecutor.getInstance(conf));
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Value of CGroupsHandler is: " + cGroupsHandler);
+          }
         }
       }
     }
@@ -306,16 +309,32 @@ public class ResourceHandlerModule {
       List<ResourceHandler> handlerList, Configuration conf,
       Context nmContext) throws ResourceHandlerException {
     ResourcePluginManager pluginManager = nmContext.getResourcePluginManager();
-    if (pluginManager != null) {
+
-       Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
+    if (pluginManager == null) {
-       if (pluginMap != null) {
+      LOG.warn("Plugin manager was null while trying to add " +
-        for (ResourcePlugin plugin : pluginMap.values()) {
+          "ResourceHandlers from configuration!");
-          addHandlerIfNotNull(handlerList, plugin
+      return;
-              .createResourceHandler(nmContext,
+    }
-                  getInitializedCGroupsHandler(conf),
+
-                  PrivilegedOperationExecutor.getInstance(conf)));
+    Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
-        }
+    if (pluginMap == null) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("List of plugins of ResourcePluginManager was empty " +
+            "while trying to add ResourceHandlers from configuration!");
       }
+      return;
+    } else {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("List of plugins of ResourcePluginManager: " +
+            pluginManager.getNameToPlugins());
+      }
+    }
+
+    for (ResourcePlugin plugin : pluginMap.values()) {
+      addHandlerIfNotNull(handlerList,
+          plugin.createResourceHandler(nmContext,
+              getInitializedCGroupsHandler(conf),
+              PrivilegedOperationExecutor.getInstance(conf)));
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java

@@ -277,4 +277,9 @@ public class TrafficControlBandwidthHandlerImpl
 
     return null;
   }
+
+  @Override
+  public String toString() {
+    return TrafficControlBandwidthHandlerImpl.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java

@@ -55,11 +55,11 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
 
   private final String REQUEST_FPGA_IP_ID_KEY = "REQUESTED_FPGA_IP_ID";
 
-  private AbstractFpgaVendorPlugin vendorPlugin;
+  private final AbstractFpgaVendorPlugin vendorPlugin;
 
-  private FpgaResourceAllocator allocator;
+  private final FpgaResourceAllocator allocator;
 
-  private CGroupsHandler cGroupsHandler;
+  private final CGroupsHandler cGroupsHandler;
 
   public static final String EXCLUDED_FPGAS_CLI_OPTION = "--excluded_fpgas";
   public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
@@ -217,4 +217,12 @@ public class FpgaResourceHandlerImpl implements ResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return FpgaResourceHandlerImpl.class.getName() + "{" +
+        "vendorPlugin=" + vendorPlugin +
+        ", allocator=" + allocator +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java

@@ -301,4 +301,9 @@ public class GpuResourceAllocator {
     }
     return assigns;
   }
+
+  @Override
+  public String toString() {
+    return GpuResourceAllocator.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java

@@ -48,9 +48,9 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
   public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
   public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
 
-  private GpuResourceAllocator gpuAllocator;
+  private final GpuResourceAllocator gpuAllocator;
-  private CGroupsHandler cGroupsHandler;
+  private final CGroupsHandler cGroupsHandler;
-  private PrivilegedOperationExecutor privilegedOperationExecutor;
+  private final PrivilegedOperationExecutor privilegedOperationExecutor;
 
   public GpuResourceHandlerImpl(Context nmContext,
       CGroupsHandler cGroupsHandler,
@@ -175,4 +175,11 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return GpuResourceHandlerImpl.class.getName() + "{" +
+        "gpuAllocator=" + gpuAllocator +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceHandlerImpl.java

@@ -41,8 +41,8 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
 
   private static final Log LOG = LogFactory
       .getLog(NumaResourceHandlerImpl.class);
-  private NumaResourceAllocator numaResourceAllocator;
+  private final NumaResourceAllocator numaResourceAllocator;
-  private String numaCtlCmd;
+  private final String numaCtlCmd;
 
   public NumaResourceHandlerImpl(Configuration conf, Context nmContext) {
     LOG.info("NUMA resources allocation is enabled, initializing NUMA resources"
@@ -105,4 +105,12 @@ public class NumaResourceHandlerImpl implements ResourceHandler {
   public List<PrivilegedOperation> teardown() throws ResourceHandlerException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return NumaResourceHandlerImpl.class.getName() + "{" +
+        "numaResourceAllocator=" + numaResourceAllocator +
+        ", numaCtlCmd='" + numaCtlCmd + '\'' +
+        '}';
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java

@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -52,7 +53,13 @@ public class ResourcePluginManager {
   public synchronized void initialize(Context context)
       throws YarnException {
     Configuration conf = context.getConf();
+
     String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
+    if (plugins == null || plugins.length == 0) {
+      LOG.info("No Resource plugins found from configuration!");
+    }
+    LOG.info("Found Resource plugins from configuration: "
+        + Arrays.toString(plugins));
 
     if (plugins != null) {
       Map<String, ResourcePlugin> pluginMap = new HashMap<>();
@@ -64,23 +71,21 @@ public class ResourcePluginManager {
           String msg =
               "Trying to initialize resource plugin with name=" + resourceName
                   + ", it is not supported, list of supported plugins:"
-                  + StringUtils.join(",",
+                  + StringUtils.join(",", SUPPORTED_RESOURCE_PLUGINS);
-                  SUPPORTED_RESOURCE_PLUGINS);
           LOG.error(msg);
           throw new YarnException(msg);
         }
 
         if (pluginMap.containsKey(resourceName)) {
-          // Duplicated items, ignore ...
+          LOG.warn("Ignoring duplicate Resource plugin definition: " +
+              resourceName);
           continue;
         }
 
         ResourcePlugin plugin = null;
         if (resourceName.equals(GPU_URI)) {
           plugin = new GpuResourcePlugin();
-        }
+        } else if (resourceName.equals(FPGA_URI)) {
-
-        if (resourceName.equals(FPGA_URI)) {
           plugin = new FpgaResourcePlugin();
         }
 
@@ -90,6 +95,7 @@ public class ResourcePluginManager {
                   + " should be loaded and initialized");
         }
         plugin.initialize(context);
+        LOG.info("Initialized plugin {}", plugin);
         pluginMap.put(resourceName, plugin);
       }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaResourcePlugin.java

@@ -102,4 +102,9 @@ public class FpgaResourcePlugin implements ResourcePlugin {
   public NMResourceInfo getNMResourceInfo() throws YarnException {
     return null;
   }
+
+  @Override
+  public String toString() {
+    return FpgaResourcePlugin.class.getName();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java

@@ -89,4 +89,9 @@ public class GpuResourcePlugin implements ResourcePlugin {
     return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus,
         assignedGpuDevices);
   }
+
+  @Override
+  public String toString() {
+    return GpuResourcePlugin.class.getName();
+  }
 }