Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8934. [GPG] Add JvmMetricsInfo and pause monitor. Contributed by Bilwa S T.

This commit is contained in:
Botong Huang 2018-11-28 15:04:30 -08:00
parent e1017a676b
commit d93507ef15
11 changed files with 386 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -3430,6 +3430,14 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_GPG_PREFIX =
FEDERATION_PREFIX + "gpg.";
public static final String GPG_WEBAPP_PREFIX = FEDERATION_GPG_PREFIX
+ "webapp.";
/** Enable/disable CORS filter. */
public static final String GPG_WEBAPP_ENABLE_CORS_FILTER =
GPG_WEBAPP_PREFIX + "cross-origin.enabled";
public static final boolean DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER = false;
// The number of threads to use for the GPG scheduled executor service
public static final String GPG_SCHEDULED_EXECUTOR_THREADS =
FEDERATION_GPG_PREFIX + "scheduled.executor.threads";
@ -3457,6 +3465,22 @@ public class YarnConfiguration extends Configuration {
FEDERATION_GPG_PREFIX + "application.cleaner.interval-ms";
public static final long DEFAULT_GPG_APPCLEANER_INTERVAL_MS = -1;
/** The address of the GPG web application. */
public static final String GPG_WEBAPP_ADDRESS =
GPG_WEBAPP_PREFIX + "address";
public static final int DEFAULT_GPG_WEBAPP_PORT = 8069;
public static final String DEFAULT_GPG_WEBAPP_ADDRESS =
"0.0.0.0:" + DEFAULT_GPG_WEBAPP_PORT;
/** The https address of the GPG web application. */
public static final String GPG_WEBAPP_HTTPS_ADDRESS =
GPG_WEBAPP_PREFIX + "https.address";
public static final int DEFAULT_GPG_WEBAPP_HTTPS_PORT = 8070;
public static final String DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_GPG_WEBAPP_HTTPS_PORT;
/**
* Specifications on how (many times) to contact Router for apps. We need to
* do this because Router might return partial application list because some

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
View File

@ -237,6 +237,7 @@
<exclude>src/main/resources/webapps/test/.keep</exclude>
<exclude>src/main/resources/webapps/proxy/.keep</exclude>
<exclude>src/main/resources/webapps/node/.keep</exclude>
<exclude>src/main/resources/webapps/gpg/.keep</exclude>
<exclude>src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css</exclude>
<exclude>src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css</exclude>
<exclude>src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css</exclude>

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View File

@ -143,6 +143,16 @@ public class WebAppUtils {
}
}
public static String getGPGWebAppURLWithoutScheme(Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
return conf.get(YarnConfiguration.GPG_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_GPG_WEBAPP_HTTPS_ADDRESS);
} else {
return conf.get(YarnConfiguration.GPG_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_GPG_WEBAPP_ADDRESS);
}
}
public static List<String> getProxyHostsAndPortsForAmFilter(
Configuration conf) {
List<String> addrs = new ArrayList<String>();

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3705,6 +3705,34 @@
<value>3600000</value>
</property>
<property>
<description>Flag to enable cross-origin (CORS) support in the GPG. This flag
requires the CORS filter initializer to be added to the filter initializers
list in core-site.xml.</description>
<name>yarn.federation.gpg.webapp.cross-origin.enabled</name>
<value>false</value>
</property>
<property>
<description>
The http address of the GPG web application.
If only a host is provided as the value,
the webapp will be served on a random port.
</description>
<name>yarn.federation.gpg.webapp.address</name>
<value>0.0.0.0:8069</value>
</property>
<property>
<description>
The https address of the GPG web application.
If only a host is provided as the value,
the webapp will be served on a random port.
</description>
<name> yarn.federation.gpg.webapp.https.address</name>
<value>0.0.0.0:8070</value>
</property>
<property>
<description>
The configured policy generator class, runs NoOpGlobalPolicy by default

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.globalpolicygenerator;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -25,9 +27,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang3.time.DurationFormatUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
@ -37,9 +43,15 @@ import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade
import org.apache.hadoop.yarn.server.globalpolicygenerator.applicationcleaner.ApplicationCleaner;
import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator;
import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner;
import org.apache.hadoop.yarn.server.globalpolicygenerator.webapp.GPGWebApp;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
/**
* Global Policy Generator (GPG) is a Yarn Federation component. By tuning the
* Federation policies in Federation State Store, GPG overlooks the entire
@ -60,6 +72,7 @@ public class GlobalPolicyGenerator extends CompositeService {
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
private AtomicBoolean isStopping = new AtomicBoolean(false);
private static final String METRICS_NAME = "Global Policy Generator";
private static long gpgStartupTime = System.currentTimeMillis();
// Federation Variables
private GPGContext gpgContext;
@ -70,6 +83,9 @@ public class GlobalPolicyGenerator extends CompositeService {
private SubClusterCleaner subClusterCleaner;
private ApplicationCleaner applicationCleaner;
private PolicyGenerator policyGenerator;
private String webAppAddress;
private JvmPauseMonitor pauseMonitor;
private WebApp webApp;
public GlobalPolicyGenerator() {
super(GlobalPolicyGenerator.class.getName());
@ -110,7 +126,13 @@ public class GlobalPolicyGenerator extends CompositeService {
this.policyGenerator = new PolicyGenerator(conf, this.gpgContext);
this.webAppAddress = WebAppUtils.getGPGWebAppURLWithoutScheme(conf);
DefaultMetricsSystem.initialize(METRICS_NAME);
JvmMetrics jm = JvmMetrics.initSingleton("GPG", null);
pauseMonitor = new JvmPauseMonitor();
addService(pauseMonitor);
jm.setPauseMonitor(pauseMonitor);
// super.serviceInit after all services are added
super.serviceInit(conf);
@ -154,6 +176,7 @@ public class GlobalPolicyGenerator extends CompositeService {
LOG.info("Scheduled policygenerator with interval: {}",
DurationFormatUtils.formatDurationISO(policyGeneratorIntervalMillis));
}
startWepApp();
}
@Override
@ -176,6 +199,9 @@ public class GlobalPolicyGenerator extends CompositeService {
if (this.isStopping.getAndSet(true)) {
return;
}
if (webApp != null) {
webApp.stop();
}
DefaultMetricsSystem.shutdown();
super.serviceStop();
}
@ -202,6 +228,42 @@ public class GlobalPolicyGenerator extends CompositeService {
this.start();
}
@VisibleForTesting
public void startWepApp() {
boolean enableCors = getConfig().getBoolean(
YarnConfiguration.GPG_WEBAPP_ENABLE_CORS_FILTER,
YarnConfiguration.DEFAULT_GPG_WEBAPP_ENABLE_CORS_FILTER);
if (enableCors) {
getConfig().setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+ HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
}
// Always load pseudo authentication filter to parse "user.name" in an URL
// to identify a HTTP request's user.
boolean hasHadoopAuthFilterInitializer = false;
String filterInitializerConfKey = "hadoop.http.filter.initializers";
Class<?>[] initializersClasses = getConfig()
.getClasses(filterInitializerConfKey);
List<String> targets = new ArrayList<String>();
if (initializersClasses != null) {
for (Class<?> initializer : initializersClasses) {
if (initializer.getName()
.equals(AuthenticationFilterInitializer.class.getName())) {
hasHadoopAuthFilterInitializer = true;
break;
}
targets.add(initializer.getName());
}
}
if (!hasHadoopAuthFilterInitializer) {
targets.add(AuthenticationFilterInitializer.class.getName());
getConfig().set(filterInitializerConfKey, StringUtils.join(",", targets));
}
LOG.info("Instantiating GPGWebApp at " + webAppAddress);
GPGWebApp gpgWebApp = new GPGWebApp(this);
webApp = WebApps.$for("gpg").at(webAppAddress).start(gpgWebApp);
}
@SuppressWarnings("resource")
public static void startGPG(String[] argv, Configuration conf) {
boolean federationEnabled =
@ -225,6 +287,10 @@ public class GlobalPolicyGenerator extends CompositeService {
}
}
public static long getGPGStartupTime() {
return gpgStartupTime;
}
public static void main(String[] argv) {
startGPG(argv, new YarnConfiguration());
}

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import org.apache.hadoop.yarn.webapp.Controller;
import com.google.inject.Inject;
/**
* Controller for the GPG Web UI.
*/
public class GPGController extends Controller {
@Inject
GPGController(RequestContext ctx) {
super(ctx);
}
@Override
public void index() {
setTitle("GPG");
render(GPGOverviewPage.class);
}
public void overview() {
setTitle("GPG Details");
render(GPGOverviewPage.class);
}
}

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewBlock.java Normal file
View File

@ -0,0 +1,50 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import java.util.Date;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
/**
* Overview block for the GPG Web UI.
*/
public class GPGOverviewBlock extends HtmlBlock {
@Inject
GPGOverviewBlock(GlobalPolicyGenerator gpg, ViewContext ctx) {
super(ctx);
}
@Override
protected void render(Block html) {
info("GPG Details")
.__("GPG started on",
new Date(GlobalPolicyGenerator.getGPGStartupTime()))
.__("GPG Version", YarnVersionInfo.getVersion())
.__("Hadoop Version", VersionInfo.getVersion());
html.__(InfoBlock.class);
}
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGOverviewPage.java Normal file
View File

@ -0,0 +1,52 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
/**
* Overview page for the GPG Web UI.
*/
public class GPGOverviewPage extends TwoColumnLayout {
@Override
protected void preHead(Page.HTML<__> html) {
commonPreHead(html);
setTitle("GPG");
}
protected void commonPreHead(Page.HTML<__> html) {
set(ACCORDION_ID, "nav");
set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
}
@Override
protected Class<? extends SubView> nav() {
return NavBlock.class;
}
@Override
protected Class<? extends SubView> content() {
return GPGOverviewBlock.class;
}
}

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java Normal file
View File

@ -0,0 +1,45 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
/**
* The GPG webapp.
*/
public class GPGWebApp extends WebApp{
private GlobalPolicyGenerator gpg;
public GPGWebApp(GlobalPolicyGenerator gpg) {
this.gpg = gpg;
}
@Override
public void setup() {
bind(JAXBContextResolver.class);
bind(GPGWebApp.class).toInstance(this);
bind(GenericExceptionHandler.class);
if (gpg != null) {
bind(GlobalPolicyGenerator.class).toInstance(gpg);
}
route("/", GPGController.class, "overview");
}
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java Normal file
View File

@ -0,0 +1,42 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
/**
* Navigation block for the GPG Web UI.
*/
public class NavBlock extends HtmlBlock {
@Override
public void render(Block html) {
html.
div("#nav").
h3("GPG").
ul().
li().a(url(""), "Overview").__().
__().
h3("Tools").
ul().
li().a("/conf", "Configuration").__().
li().a("/logs", "Local logs").__().
li().a("/stacks", "Server stacks").__().
li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
}
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/package-info.java Normal file
View File

@ -0,0 +1,24 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Classes comprising the policy generator for the GPG. Responsibilities include
* generating and updating policies based on the cluster status.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;