From 80557b872fbf31fec94930fc4723d7a459fe52e7 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Mon, 2 Jun 2014 09:19:16 -0700 Subject: [PATCH] HBASE-10289 Avoid random port usage by default JMX Server. Create Custome JMX server (Qiang Tian) --- conf/hbase-env.sh | 4 +- .../org/apache/hadoop/hbase/JMXListener.java | 195 ++++++++++++++++++ .../apache/hadoop/hbase/TestJMXListener.java | 95 +++++++++ src/main/docbkx/configuration.xml | 107 +++++++++- 4 files changed, 394 insertions(+), 7 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh index e6d1c9c90ff..9059d12e27e 100644 --- a/conf/hbase-env.sh +++ b/conf/hbase-env.sh @@ -76,7 +76,9 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC" # Uncomment and adjust to enable JMX exporting # See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. # More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# +# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX +# section in HBase Reference Guide for instructions. + # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" # export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" # export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java new file mode 100644 index 00000000000..7a1ea11d6b8 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java @@ -0,0 +1,195 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.*; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.rmi.registry.LocateRegistry; +import java.rmi.server.RMIClientSocketFactory; +import java.rmi.server.RMIServerSocketFactory; +import java.util.HashMap; + +import javax.management.MBeanServer; +import javax.management.remote.JMXConnectorServer; +import javax.management.remote.JMXConnectorServerFactory; +import javax.management.remote.JMXServiceURL; +import javax.management.remote.rmi.RMIConnectorServer; +import javax.rmi.ssl.SslRMIClientSocketFactory; +import javax.rmi.ssl.SslRMIServerSocketFactory; + +/** + * Pluggable JMX Agent for HBase(to fix the 2 random TCP ports issue + * of the out-of-the-box JMX Agent): + * 1)connector port can share with the registry port if SSL is OFF + * 2)support password authentication + * 3)support subset of SSL (with default configuration) + */ +public class JMXListener implements Coprocessor { + + public static final Log LOG = LogFactory.getLog(JMXListener.class); + public static final String RMI_REGISTRY_PORT_CONF_KEY = ".rmi.registry.port"; + public static final String RMI_CONNECTOR_PORT_CONF_KEY = ".rmi.connector.port"; + public static int defRMIRegistryPort = 10102; + + /** + * workaround for HBASE-11146 + * master and regionserver are in 1 JVM in standalone mode + * only 1 JMX instance is allowed, otherwise there is port conflict even if + * we only load regionserver coprocessor on master + */ + private static JMXConnectorServer jmxCS = null; + + public static JMXServiceURL buildJMXServiceURL(int rmiRegistryPort, + int rmiConnectorPort) throws IOException { + // Build jmxURL + StringBuilder url = new StringBuilder(); + url.append("service:jmx:rmi://localhost:"); + url.append(rmiConnectorPort); + url.append("/jndi/rmi://localhost:"); + url.append(rmiRegistryPort); + url.append("/jmxrmi"); + + return new JMXServiceURL(url.toString()); + + } + + public void startConnectorServer(int rmiRegistryPort, int rmiConnectorPort) + throws IOException { + boolean rmiSSL = false; + boolean authenticate = true; + String passwordFile = null; + String accessFile = null; + + System.setProperty("java.rmi.server.randomIDs", "true"); + + String rmiSSLValue = System.getProperty("com.sun.management.jmxremote.ssl", + "false"); + rmiSSL = Boolean.parseBoolean(rmiSSLValue); + + String authenticateValue = + System.getProperty("com.sun.management.jmxremote.authenticate", "false"); + authenticate = Boolean.parseBoolean(authenticateValue); + + passwordFile = System.getProperty("com.sun.management.jmxremote.password.file"); + accessFile = System.getProperty("com.sun.management.jmxremote.access.file"); + + LOG.info("rmiSSL:" + rmiSSLValue + ",authenticate:" + authenticateValue + + ",passwordFile:" + passwordFile + ",accessFile:" + accessFile); + + // Environment map + HashMap jmxEnv = new HashMap(); + + RMIClientSocketFactory csf = null; + RMIServerSocketFactory ssf = null; + + if (rmiSSL) { + if (rmiRegistryPort == rmiConnectorPort) { + throw new IOException("SSL is enabled. " + + "rmiConnectorPort cannot share with the rmiRegistryPort!"); + } + csf = new SslRMIClientSocketFactory(); + ssf = new SslRMIServerSocketFactory(); + } + + if (csf != null) { + jmxEnv.put(RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE, csf); + } + if (ssf != null) { + jmxEnv.put(RMIConnectorServer.RMI_SERVER_SOCKET_FACTORY_ATTRIBUTE, ssf); + } + + // Configure authentication + if (authenticate) { + jmxEnv.put("jmx.remote.x.password.file", passwordFile); + jmxEnv.put("jmx.remote.x.access.file", accessFile); + } + + // Create the RMI registry + LocateRegistry.createRegistry(rmiRegistryPort); + // Retrieve the PlatformMBeanServer. + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + + // Build jmxURL + JMXServiceURL serviceUrl = buildJMXServiceURL(rmiRegistryPort, rmiConnectorPort); + + try { + // Start the JMXListener with the connection string + jmxCS = JMXConnectorServerFactory.newJMXConnectorServer(serviceUrl, jmxEnv, mbs); + jmxCS.start(); + LOG.info("ConnectorServer started!"); + } catch (IOException e) { + LOG.error("fail to start connector server!", e); + } + + } + + public void stopConnectorServer() throws IOException { + synchronized(JMXListener.class) { + if (jmxCS != null) { + jmxCS.stop(); + LOG.info("ConnectorServer stopped!"); + jmxCS = null; + } + } + } + + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + int rmiRegistryPort = -1; + int rmiConnectorPort = -1; + Configuration conf = env.getConfiguration(); + + if (env instanceof MasterCoprocessorEnvironment) { + LOG.error("JMXListener should not be loaded in Master Environment!"); + } else if (env instanceof RegionServerCoprocessorEnvironment) { + // running on RegionServer --since 0.99 HMaster is also a HRegionServer + rmiRegistryPort = + conf.getInt("regionserver" + RMI_REGISTRY_PORT_CONF_KEY, defRMIRegistryPort); + rmiConnectorPort = + conf.getInt("regionserver" + RMI_CONNECTOR_PORT_CONF_KEY, rmiRegistryPort); + LOG.info("RegionServer rmiRegistryPort:" + rmiRegistryPort + + ",RegionServer rmiConnectorPort:" + rmiConnectorPort); + + } else if (env instanceof RegionCoprocessorEnvironment) { + LOG.error("JMXListener should not be loaded in Region Environment!"); + } + + synchronized(JMXListener.class) { + if (jmxCS != null) { + LOG.info("JMXListener has been started at Registry port " + rmiRegistryPort); + } + else { + startConnectorServer(rmiRegistryPort, rmiConnectorPort); + } + } + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { + stopConnectorServer(); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java new file mode 100644 index 00000000000..719e04f0372 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java @@ -0,0 +1,95 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import javax.management.MBeanServerConnection; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.ExpectedException; + + + +@Category(MediumTests.class) +public class TestJMXListener { + private static final Log LOG = LogFactory.getLog(TestJMXListener.class); + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static int connectorPort = 61120; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + + conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, + JMXListener.class.getName()); + conf.setInt("regionserver.rmi.registry.port", connectorPort); + + UTIL.startMiniCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testStart() throws Exception { + JMXConnector connector = JMXConnectorFactory.connect( + JMXListener.buildJMXServiceURL(connectorPort,connectorPort)); + + MBeanServerConnection mb = connector.getMBeanServerConnection(); + String domain = mb.getDefaultDomain(); + Assert.assertTrue("default domain is not correct", + !domain.isEmpty()); + connector.close(); + + } + + //shutdown hbase only. then try connect, IOException expected + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + @Test + public void testStop() throws Exception { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + LOG.info("shutdown hbase cluster..."); + cluster.shutdown(); + LOG.info("wait for the hbase cluster shutdown..."); + cluster.waitUntilShutDown(); + + JMXConnector connector = JMXConnectorFactory.newJMXConnector( + JMXListener.buildJMXServiceURL(connectorPort,connectorPort), null); + expectedEx.expect(IOException.class); + connector.connect(); + + } + + +} \ No newline at end of file diff --git a/src/main/docbkx/configuration.xml b/src/main/docbkx/configuration.xml index 1f4228a1cd2..2fbc3e83b07 100644 --- a/src/main/docbkx/configuration.xml +++ b/src/main/docbkx/configuration.xml @@ -1411,7 +1411,8 @@ index e70ebc6..96f8c27 100644 a late-version HDFS so you have the fixes he refers too and himself adds to HDFS that help HBase MTTR (e.g. HDFS-3703, HDFS-3712, and HDFS-4791 -- hadoop 2 for sure has them and late hadoop 1 has some). Set the following in the RegionServer. - + hbase.lease.recovery.dfs.timeout 23000 How much time we allow elapse between calls to recover lease. @@ -1421,12 +1422,13 @@ index e70ebc6..96f8c27 100644 dfs.client.socket-timeout 10000 Down the DFS timeout from 60 to 10 seconds. -]]> - + +]]> And on the namenode/datanode side, set the following to enable 'staleness' introduced in HDFS-3703, HDFS-3912. - + dfs.client.socket-timeout 10000 Down the DFS timeout from 60 to 10 seconds. @@ -1460,12 +1462,105 @@ index e70ebc6..96f8c27 100644 dfs.namenode.avoid.write.stale.datanode true Enable stale state in hdfs -]]> - + +]]> +
+ JMX + JMX(Java Management Extensions) provides built-in instrumentation that enables you + to monitor and manage the Java VM. To enable monitoring and management from remote + systems, you need to set system property com.sun.management.jmxremote.port(the port + number through which you want to enable JMX RMI connections) when you start the Java VM. + See + official document for more information. Historically, besides above port mentioned, + JMX opens 2 additional random TCP listening ports, which could lead to port conflict + problem.(See HBASE-10289 + for details) + + As an alternative, You can use the coprocessor-based JMX implementation provided + by HBase. To enable it, add below property in hbase-site.xml: + + hbase.coprocessor.regionserver.classes + org.apache.hadoop.hbase.JMXListener + +]]> + NOTE: DO NOT set com.sun.management.jmxremote.port for Java VM at the same time. + + Currently it supports Master and RegionServer Java VM. The reason why you only + configure coprocessor for 'regionserver' is that, starting from HBase 0.99, + a Master IS also a RegionServer. (See HBASE-10569 + for more information.) + By default, the JMX listens on TCP port 10102, you can further configure the port + using below properties: + + + regionserver.rmi.registry.port + 61130 + + + regionserver.rmi.connector.port + 61140 + +]]> + The registry port can be shared with connector port in most cases, so you only + need to configure regionserver.rmi.registry.port. However if you want to use SSL + communication, the 2 ports must be configured to different values. + + + By default the password authentication and SSL communication is disabled. + To enable password authentication, you need to update hbase-env.sh + like below: + +export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.authenticate=true \ + -Dcom.sun.management.jmxremote.password.file=your_password_file \ + -Dcom.sun.management.jmxremote.access.file=your_access_file" + +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE " +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE " + + See example password/access file under $JRE_HOME/lib/management. + + + To enable SSL communication with password authentication, follow below steps: + +#1. generate a key pair, stored in myKeyStore +keytool -genkey -alias jconsole -keystore myKeyStore + +#2. export it to file jconsole.cert +keytool -export -alias jconsole -keystore myKeyStore -file jconsole.cert + +#3. copy jconsole.cert to jconsole client machine, import it to jconsoleKeyStore +keytool -import -alias jconsole -keystore jconsoleKeyStore -file jconsole.cert + + And then update hbase-env.sh like below: + +export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=true \ + -Djavax.net.ssl.keyStore=/home/tianq/myKeyStore \ + -Djavax.net.ssl.keyStorePassword=your_password_in_step_#1 \ + -Dcom.sun.management.jmxremote.authenticate=true \ + -Dcom.sun.management.jmxremote.password.file=your_password file \ + -Dcom.sun.management.jmxremote.access.file=your_access_file" + +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE " +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE " + + + Finally start jconsole on client using the key store: + +jconsole -J-Djavax.net.ssl.trustStore=/home/tianq/jconsoleKeyStore + +
+ +