diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 128ce79a294..c4dfd2f9d7c 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -584,6 +584,12 @@ ${hadoop.version} + + org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + ${project.version} + + org.apache.hadoop hadoop-yarn-services-core diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 7fff145c03b..8e86b26cf2f 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -39,6 +39,7 @@ function hadoop_usage hadoop_add_subcommand "container" client "prints container(s) report" hadoop_add_subcommand "daemonlog" admin "get/set the log level for each daemon" hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables" + hadoop_add_subcommand "globalpolicygenerator" daemon "run the Global Policy Generator" hadoop_add_subcommand "jar " client "run a jar file" hadoop_add_subcommand "logs" client "dump container logs" hadoop_add_subcommand "node" admin "prints node report(s)" @@ -106,6 +107,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'" exit 0 ;; + globalpolicygenerator) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator' + ;; jar) HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar ;; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 03c66b09edf..5207de816bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -134,6 +134,10 @@ if "%1" == "--loglevel" ( set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-router\target\classes ) + if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-globalpolicygenerator\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-globalpolicygenerator\target\classes + ) + if exist %HADOOP_YARN_HOME%\build\test\classes ( set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\test\classes ) @@ -155,7 +159,7 @@ if "%1" == "--loglevel" ( set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^ application applicationattempt container node queue logs daemonlog historyserver ^ - timelineserver timelinereader router classpath + timelineserver timelinereader router globalpolicygenerator classpath for %%i in ( %yarncommands% ) do ( if %yarn-command% == %%i set yarncommand=true ) @@ -265,6 +269,12 @@ goto :eof ) goto :eof +:globalpolicygenerator + set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\globalpolicygenerator-config\log4j.properties + set CLASS=org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator + set YARN_OPTS=%YARN_OPTS% %YARN_GLOBALPOLICYGENERATOR_OPTS% + goto :eof + :routeradmin set CLASS=org.apache.hadoop.yarn.client.cli.RouterCLI set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% @@ -347,6 +357,7 @@ goto :eof @echo resourcemanager run the ResourceManager @echo nodemanager run a nodemanager on each slave @echo router run the Router daemon + @echo globalpolicygenerator run the Global Policy Generator @echo routeradmin router admin tools @echo timelineserver run the timeline server @echo timelinereader run the timeline reader server diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh index b3650a6120c..b6c2aaf0942 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh @@ -160,6 +160,18 @@ # #export YARN_ROUTER_OPTS= +### +# Global Policy Generator specific parameters +### + +# Specify the JVM options to be used when starting the GPG. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# See ResourceManager for some examples +# +#export YARN_GLOBALPOLICYGENERATOR_OPTS= + ### # Registry DNS specific parameters # This is deprecated and should be done in hadoop-env.sh diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml new file mode 100644 index 00000000000..d3fe7d9cc02 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml @@ -0,0 +1,99 @@ + + + + + hadoop-yarn-server + org.apache.hadoop + 3.4.0-SNAPSHOT + + 4.0.0 + org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + 3.4.0-SNAPSHOT + hApache Hadoop YARN GlobalPolicyGenerator + + + + ${project.parent.parent.basedir} + + + + + + org.apache.hadoop + hadoop-yarn-server-common + + + + org.apache.hadoop + hadoop-common + + + + org.apache.hadoop + hadoop-yarn-api + + + + org.apache.hadoop + hadoop-yarn-common + + + + org.apache.hadoop + hadoop-common + test-jar + test + + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + + junit + junit + test + + + + org.apache.hadoop + hadoop-yarn-server-common + test-jar + test + + + + org.hsqldb + hsqldb + test + jdk8 + + + + + + + + org.apache.rat + apache-rat-plugin + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java new file mode 100644 index 00000000000..da8a383bd75 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; + +/** + * Context for Global Policy Generator. + */ +public interface GPGContext { + + FederationStateStoreFacade getStateStoreFacade(); + + void setStateStoreFacade(FederationStateStoreFacade facade); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java new file mode 100644 index 00000000000..3884ace9cef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; + +/** + * Context implementation for Global Policy Generator. + */ +public class GPGContextImpl implements GPGContext { + + private FederationStateStoreFacade facade; + + @Override + public FederationStateStoreFacade getStateStoreFacade() { + return facade; + } + + @Override + public void setStateStoreFacade( + FederationStateStoreFacade federationStateStoreFacade) { + this.facade = federationStateStoreFacade; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java new file mode 100644 index 00000000000..01f4a4c41a6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Global Policy Generator (GPG) is a Yarn Federation component. By tuning the + * Federation policies in Federation State Store, GPG overlooks the entire + * federated cluster and ensures that the system is tuned and balanced all the + * time. + * + * The GPG operates continuously but out-of-band from all cluster operations, + * that allows to enforce global invariants, affect load balancing, trigger + * draining of sub-clusters that will undergo maintenance, etc. + */ +public class GlobalPolicyGenerator extends CompositeService { + + public static final Logger LOG = + LoggerFactory.getLogger(GlobalPolicyGenerator.class); + + // YARN Variables + private static CompositeServiceShutdownHook gpgShutdownHook; + public static final int SHUTDOWN_HOOK_PRIORITY = 30; + private AtomicBoolean isStopping = new AtomicBoolean(false); + private static final String METRICS_NAME = "Global Policy Generator"; + + // Federation Variables + private GPGContext gpgContext; + + public GlobalPolicyGenerator() { + super(GlobalPolicyGenerator.class.getName()); + this.gpgContext = new GPGContextImpl(); + } + + protected void initAndStart(Configuration conf, boolean hasToReboot) { + // Remove the old hook if we are rebooting. + if (hasToReboot && null != gpgShutdownHook) { + ShutdownHookManager.get().removeShutdownHook(gpgShutdownHook); + } + gpgShutdownHook = new CompositeServiceShutdownHook(this); + ShutdownHookManager.get().addShutdownHook(gpgShutdownHook, + SHUTDOWN_HOOK_PRIORITY); + this.init(conf); + this.start(); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + // Set up the context + this.gpgContext + .setStateStoreFacade(FederationStateStoreFacade.getInstance()); + + DefaultMetricsSystem.initialize(METRICS_NAME); + + // super.serviceInit after all services are added + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.isStopping.getAndSet(true)) { + return; + } + DefaultMetricsSystem.shutdown(); + super.serviceStop(); + } + + public String getName() { + return "FederationGlobalPolicyGenerator"; + } + + public GPGContext getGPGContext() { + return this.gpgContext; + } + + @SuppressWarnings("resource") + public static void startGPG(String[] argv, Configuration conf) { + boolean federationEnabled = conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED, + YarnConfiguration.DEFAULT_FEDERATION_ENABLED); + if (federationEnabled) { + Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); + StringUtils.startupShutdownMessage(GlobalPolicyGenerator.class, argv, LOG); + GlobalPolicyGenerator globalPolicyGenerator = new GlobalPolicyGenerator(); + globalPolicyGenerator.initAndStart(conf, false); + } else { + LOG.warn("Federation is not enabled. The gpg cannot start."); + } + } + + public static void main(String[] argv) { + try { + startGPG(argv, new YarnConfiguration()); + } catch (Throwable t) { + LOG.error("Error starting global policy generator", t); + System.exit(-1); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java new file mode 100644 index 00000000000..abaa57c81e0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java new file mode 100644 index 00000000000..f657b86c836 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.globalpolicygenerator; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; + +/** + * Unit test for GlobalPolicyGenerator. + */ +public class TestGlobalPolicyGenerator { + + @Test(timeout = 1000) + public void testNonFederation() { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false); + + // If GPG starts running, this call will not return + GlobalPolicyGenerator.startGPG(new String[0], conf); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml index d3e0f2d61e7..83c43feb114 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -47,5 +47,6 @@ hadoop-yarn-server-timelineservice-hbase-tests hadoop-yarn-server-router hadoop-yarn-server-timelineservice-documentstore + hadoop-yarn-server-globalpolicygenerator diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 298fa597e99..241e3bc237a 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -77,6 +77,10 @@ org.apache.hadoop hadoop-yarn-server-router + + org.apache.hadoop + hadoop-yarn-server-globalpolicygenerator + org.apache.hadoop hadoop-yarn-services-core