diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c985d7b6655..6a5e0c3d695 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -451,6 +451,12 @@ ${hadoop.version} + + org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + ${project.version} + + org.apache.hadoop hadoop-yarn-services-core diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 3ec03119a5c..0c320dcc066 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -39,6 +39,7 @@ function hadoop_usage hadoop_add_subcommand "container" client "prints container(s) report" hadoop_add_subcommand "daemonlog" admin "get/set the log level for each daemon" hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables" + hadoop_add_subcommand "globalpolicygenerator" daemon "run the Global Policy Generator" hadoop_add_subcommand "jar " client "run a jar file" hadoop_add_subcommand "logs" client "dump container logs" hadoop_add_subcommand "node" admin "prints node report(s)" @@ -104,6 +105,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'" exit 0 ;; + globalpolicygenerator) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator' + ;; jar) HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar ;; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index e1ac11287fe..bebfd71756b 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -134,6 +134,10 @@ if "%1" == "--loglevel" ( set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-router\target\classes ) + if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-globalpolicygenerator\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-globalpolicygenerator\target\classes + ) + if exist %HADOOP_YARN_HOME%\build\test\classes ( set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\test\classes ) @@ -155,7 +159,7 @@ if "%1" == "--loglevel" ( set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^ application applicationattempt container node queue logs daemonlog historyserver ^ - timelineserver timelinereader router classpath + timelineserver timelinereader router globalpolicygenerator classpath for %%i in ( %yarncommands% ) do ( if %yarn-command% == %%i set yarncommand=true ) @@ -259,7 +263,13 @@ goto :eof :router set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\router-config\log4j.properties set CLASS=org.apache.hadoop.yarn.server.router.Router - set YARN_OPTS=%YARN_OPTS% %HADOOP_ROUTER_OPTS% + set YARN_OPTS=%YARN_OPTS% %YARN_ROUTER_OPTS% + goto :eof + +:globalpolicygenerator + set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\globalpolicygenerator-config\log4j.properties + set CLASS=org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator + set YARN_OPTS=%YARN_OPTS% %YARN_GLOBALPOLICYGENERATOR_OPTS% goto :eof :nodemanager @@ -336,27 +346,28 @@ goto :eof :print_usage @echo Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND @echo where COMMAND is one of: - @echo resourcemanager run the ResourceManager - @echo nodemanager run a nodemanager on each slave - @echo router run the Router daemon - @echo timelineserver run the timeline server - @echo timelinereader run the timeline reader server - @echo rmadmin admin tools - @echo version print the version - @echo jar ^ run a jar file - @echo application prints application(s) report/kill application - @echo applicationattempt prints applicationattempt(s) report - @echo cluster prints cluster information - @echo container prints container(s) report - @echo node prints node report(s) - @echo queue prints queue information - @echo logs dump container logs - @echo schedulerconf updates scheduler configuration - @echo classpath prints the class path needed to get the - @echo Hadoop jar and the required libraries - @echo daemonlog get/set the log level for each daemon + @echo resourcemanager run the ResourceManager + @echo nodemanager run a nodemanager on each slave + @echo router run the Router daemon + @echo globalpolicygenerator run the Global Policy Generator + @echo timelineserver run the timeline server + @echo timelinereader run the timeline reader server + @echo rmadmin admin tools + @echo version print the version + @echo jar ^ run a jar file + @echo application prints application(s) report/kill application + @echo applicationattempt prints applicationattempt(s) report + @echo cluster prints cluster information + @echo container prints container(s) report + @echo node prints node report(s) + @echo queue prints queue information + @echo logs dump container logs + @echo schedulerconf updates scheduler configuration + @echo classpath prints the class path needed to get the + @echo Hadoop jar and the required libraries + @echo daemonlog get/set the log level for each daemon @echo or - @echo CLASSNAME run the class named CLASSNAME + @echo CLASSNAME run the class named CLASSNAME @echo Most commands print help when invoked w/o parameters. endlocal diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh index 6b43416ed96..942634a4a85 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh @@ -149,6 +149,18 @@ # #export YARN_ROUTER_OPTS= +### +# Global Policy Generator specific parameters +### + +# Specify the JVM options to be used when starting the GPG. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# See ResourceManager for some examples +# +#export YARN_GLOBALPOLICYGENERATOR_OPTS= + ### # Registry DNS specific parameters # This is deprecated and should be done in hadoop-env.sh diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml new file mode 100644 index 00000000000..9bbb936e7b9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml @@ -0,0 +1,98 @@ + + + + + hadoop-yarn-server + org.apache.hadoop + 3.1.0-SNAPSHOT + + 4.0.0 + org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + 3.1.0-SNAPSHOT + hadoop-yarn-server-globalpolicygenerator + + + + ${project.parent.parent.basedir} + + + + + + org.apache.hadoop + hadoop-yarn-server-common + + + + org.apache.hadoop + hadoop-common + + + + org.apache.hadoop + hadoop-yarn-api + + + + org.apache.hadoop + hadoop-yarn-common + + + + org.apache.hadoop + hadoop-common + test-jar + test + + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + + junit + junit + test + + + + org.apache.hadoop + hadoop-yarn-server-common + test-jar + test + + + + org.hsqldb + hsqldb + test + + + + + + + + org.apache.rat + apache-rat-plugin + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java new file mode 100644 index 00000000000..da8a383bd75 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; + +/** + * Context for Global Policy Generator. + */ +public interface GPGContext { + + FederationStateStoreFacade getStateStoreFacade(); + + void setStateStoreFacade(FederationStateStoreFacade facade); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java new file mode 100644 index 00000000000..3884ace9cef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; + +/** + * Context implementation for Global Policy Generator. + */ +public class GPGContextImpl implements GPGContext { + + private FederationStateStoreFacade facade; + + @Override + public FederationStateStoreFacade getStateStoreFacade() { + return facade; + } + + @Override + public void setStateStoreFacade( + FederationStateStoreFacade federationStateStoreFacade) { + this.facade = federationStateStoreFacade; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java new file mode 100644 index 00000000000..c1f7460cfc6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Global Policy Generator (GPG) is a Yarn Federation component. By tuning the + * Federation policies in Federation State Store, GPG overlooks the entire + * federated cluster and ensures that the system is tuned and balanced all the + * time. + * + * The GPG operates continuously but out-of-band from all cluster operations, + * that allows to enforce global invariants, affect load balancing, trigger + * draining of sub-clusters that will undergo maintenance, etc. + */ +public class GlobalPolicyGenerator extends CompositeService { + + public static final Logger LOG = + LoggerFactory.getLogger(GlobalPolicyGenerator.class); + + // YARN Variables + private static CompositeServiceShutdownHook gpgShutdownHook; + public static final int SHUTDOWN_HOOK_PRIORITY = 30; + private AtomicBoolean isStopping = new AtomicBoolean(false); + private static final String METRICS_NAME = "Global Policy Generator"; + + // Federation Variables + private GPGContext gpgContext; + + public GlobalPolicyGenerator() { + super(GlobalPolicyGenerator.class.getName()); + this.gpgContext = new GPGContextImpl(); + } + + protected void initAndStart(Configuration conf, boolean hasToReboot) { + try { + // Remove the old hook if we are rebooting. + if (hasToReboot && null != gpgShutdownHook) { + ShutdownHookManager.get().removeShutdownHook(gpgShutdownHook); + } + + gpgShutdownHook = new CompositeServiceShutdownHook(this); + ShutdownHookManager.get().addShutdownHook(gpgShutdownHook, + SHUTDOWN_HOOK_PRIORITY); + + this.init(conf); + this.start(); + } catch (Throwable t) { + LOG.error("Error starting globalpolicygenerator", t); + System.exit(-1); + } + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + // Set up the context + this.gpgContext + .setStateStoreFacade(FederationStateStoreFacade.getInstance()); + + DefaultMetricsSystem.initialize(METRICS_NAME); + + // super.serviceInit after all services are added + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.isStopping.getAndSet(true)) { + return; + } + DefaultMetricsSystem.shutdown(); + super.serviceStop(); + } + + public String getName() { + return "FederationGlobalPolicyGenerator"; + } + + public GPGContext getGPGContext() { + return this.gpgContext; + } + + @SuppressWarnings("resource") + public static void startGPG(String[] argv, Configuration conf) { + boolean federationEnabled = + conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED, + YarnConfiguration.DEFAULT_FEDERATION_ENABLED); + if (federationEnabled) { + Thread.setDefaultUncaughtExceptionHandler( + new YarnUncaughtExceptionHandler()); + StringUtils.startupShutdownMessage(GlobalPolicyGenerator.class, argv, + LOG); + GlobalPolicyGenerator globalPolicyGenerator = new GlobalPolicyGenerator(); + globalPolicyGenerator.initAndStart(conf, false); + } else { + LOG.warn("Federation is not enabled. The gpg cannot start."); + } + } + + public static void main(String[] argv) { + startGPG(argv, new YarnConfiguration()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java new file mode 100644 index 00000000000..abaa57c81e0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java new file mode 100644 index 00000000000..f657b86c836 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; + +/** + * Unit test for GlobalPolicyGenerator. + */ +public class TestGlobalPolicyGenerator { + + @Test(timeout = 1000) + public void testNonFederation() { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false); + + // If GPG starts running, this call will not return + GlobalPolicyGenerator.startGPG(new String[0], conf); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml index 2f510234b34..227c4389851 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -46,5 +46,6 @@ hadoop-yarn-server-timelineservice-hbase hadoop-yarn-server-timelineservice-hbase-tests hadoop-yarn-server-router + hadoop-yarn-server-globalpolicygenerator diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 08820275659..29141dc4d94 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -78,6 +78,10 @@ org.apache.hadoop hadoop-yarn-server-router + + org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + org.apache.hadoop hadoop-yarn-services-core