From f8c72d7b3acca8285bbc3024f491c4586805be1e Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Thu, 8 Nov 2018 12:23:00 +0800 Subject: [PATCH] YARN-8880. Add configurations for pluggable plugin framework. Contributed by Zhankun Tang. --- .../hadoop/yarn/conf/YarnConfiguration.java | 24 +++- .../src/main/resources/yarn-default.xml | 19 ++++ .../resourceplugin/ResourcePluginManager.java | 34 +++++- .../TestResourcePluginManager.java | 107 ++++++++++++++++-- 4 files changed, 170 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ce38d2762b2..e88d5940a6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1605,6 +1605,28 @@ public class YarnConfiguration extends Configuration { public static final String NM_RESOURCE_PLUGINS = NM_PREFIX + "resource-plugins"; + /** + * This setting controls if pluggable device plugin framework is enabled. + * */ + @Private + public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED = + NM_PREFIX + "pluggable-device-framework.enabled"; + + /** + * The pluggable device plugin framework is disabled by default + * */ + @Private + public static final boolean DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED = + false; + + /** + * This setting contains vendor plugin class names for + * device plugin framework to load. Split by comma + * */ + @Private + public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES = + NM_PREFIX + "pluggable-device-framework.device-classes"; + /** * Prefix for gpu configurations. Work in progress: This configuration * parameter may be changed/removed in the future. @@ -1647,7 +1669,7 @@ public class YarnConfiguration extends Configuration { NVIDIA_DOCKER_V1; /** - * This setting controls end point of nvidia-docker-v1 plugin + * This setting controls end point of nvidia-docker-v1 plugin. */ @Private public static final String NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1360e73f7fd..f5493bc540a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3770,6 +3770,25 @@ false + + + This setting controls if pluggable device framework is enabled. + Disabled by default + + yarn.nodemanager.pluggable-device-framework.enabled + false + + + + + Configure vendor device plugin class name here. Comma separated. + The class must be found in CLASSPATH. The pluggable device framework will + load these classes. + + yarn.nodemanager.pluggable-device-framework.device-classes + + + When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java index f28aad206a6..e930d9665ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin; @@ -52,11 +53,10 @@ public class ResourcePluginManager { public synchronized void initialize(Context context) throws YarnException { Configuration conf = context.getConf(); + Map pluginMap = new HashMap<>(); + String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS); - if (plugins != null) { - Map pluginMap = new HashMap<>(); - // Initialize each plugins for (String resourceName : plugins) { resourceName = resourceName.trim(); @@ -92,8 +92,34 @@ public class ResourcePluginManager { plugin.initialize(context); pluginMap.put(resourceName, plugin); } + } + // Try to load pluggable device plugins + boolean puggableDeviceFrameworkEnabled = conf.getBoolean( + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + YarnConfiguration.DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED); - configuredPlugins = Collections.unmodifiableMap(pluginMap); + if (puggableDeviceFrameworkEnabled) { + initializePluggableDevicePlugins(context, conf, pluginMap); + } else { + LOG.info("The pluggable device framework is not enabled." + + " If you want, please set true to {}", + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED); + } + configuredPlugins = Collections.unmodifiableMap(pluginMap); + } + + public void initializePluggableDevicePlugins(Context context, + Configuration configuration, + Map pluginMap) + throws YarnRuntimeException { + LOG.info("The pluggable device framework enabled," + + "trying to load the vendor plugins"); + + String[] pluginClassNames = configuration.getStrings( + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); + if (null == pluginClassNames) { + throw new YarnRuntimeException("Null value found in configuration: " + + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java index 6ed7c568899..eb8bf542e9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; @@ -41,12 +42,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.util.HashMap; @@ -58,10 +57,18 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; public class TestResourcePluginManager extends NodeManagerTestBase { private NodeManager nm; + private YarnConfiguration conf; + + @Before + public void setup() throws Exception { + this.conf = createNMConfig(); + } + ResourcePluginManager stubResourcePluginmanager() { // Stub ResourcePluginManager final ResourcePluginManager rpm = mock(ResourcePluginManager.class); @@ -183,7 +190,6 @@ public class TestResourcePluginManager extends NodeManagerTestBase { final ResourcePluginManager rpm = stubResourcePluginmanager(); nm = new MyMockNM(rpm); - YarnConfiguration conf = createNMConfig(); nm.init(conf); verify(rpm, times(1)).initialize( any(Context.class)); @@ -198,7 +204,6 @@ public class TestResourcePluginManager extends NodeManagerTestBase { nm = new MyMockNM(rpm); - YarnConfiguration conf = createNMConfig(); nm.init(conf); nm.start(); @@ -238,15 +243,14 @@ public class TestResourcePluginManager extends NodeManagerTestBase { } @Override - protected ContainerExecutor createContainerExecutor(Configuration conf) { + protected ContainerExecutor createContainerExecutor( + Configuration configuration) { ((NMContext)this.getNMContext()).setResourcePluginManager(rpm); - lce.setConf(conf); + lce.setConf(configuration); return lce; } }; - YarnConfiguration conf = createNMConfig(); - nm.init(conf); nm.start(); @@ -264,4 +268,89 @@ public class TestResourcePluginManager extends NodeManagerTestBase { } Assert.assertTrue("New ResourceHandler should be added", newHandlerAdded); } + + // Disabled pluggable framework in configuration. + // We use spy object of real rpm to verify "initializePluggableDevicePlugins" + // because use mock rpm will not working + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkDisabled() + throws Exception { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + false); + nm.init(conf); + nm.start(); + verify(rpmSpy, times(1)).initialize( + any(Context.class)); + verify(rpmSpy, times(0)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + } + + // No related configuration set. + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkDisabled2() + throws Exception { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + nm.init(conf); + nm.start(); + verify(rpmSpy, times(1)).initialize( + any(Context.class)); + verify(rpmSpy, times(0)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + } + + // Enable framework and configure pluggable device classes + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkEnabled() + throws Exception { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + conf.setStrings( + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES, + "com.cmp1.hdw1plugin"); + nm.init(conf); + nm.start(); + verify(rpmSpy, times(1)).initialize( + any(Context.class)); + verify(rpmSpy, times(1)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + } + + // Enable pluggable framework, but leave device classes un-configured + // initializePluggableDevicePlugins invoked but it should throw an exception + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkEnabled2() { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + Boolean fail = false; + try { + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + + nm.init(conf); + nm.start(); + } catch (YarnRuntimeException e) { + fail = true; + } catch (Exception e) { + + } + verify(rpmSpy, times(1)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + Assert.assertTrue(fail); + } }