Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-10289 Avoid random port usage by default JMX Server. Create Custome JMX server (Qiang Tian)

This commit is contained in:
Michael Stack 2014-06-02 09:19:16 -07:00
parent d56dfd2a8b
commit 80557b872f
4 changed files with 394 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
conf/hbase-env.sh
View File

@ -76,7 +76,9 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
# Uncomment and adjust to enable JMX exporting
# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
#
# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX
# section in HBase Reference Guide for instructions.
# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"

195
hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java Normal file
View File

@ -0,0 +1,195 @@
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.*;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.rmi.registry.LocateRegistry;
import java.rmi.server.RMIClientSocketFactory;
import java.rmi.server.RMIServerSocketFactory;
import java.util.HashMap;
import javax.management.MBeanServer;
import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXConnectorServerFactory;
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
import javax.rmi.ssl.SslRMIClientSocketFactory;
import javax.rmi.ssl.SslRMIServerSocketFactory;
/**
* Pluggable JMX Agent for HBase(to fix the 2 random TCP ports issue
* of the out-of-the-box JMX Agent):
* 1)connector port can share with the registry port if SSL is OFF
* 2)support password authentication
* 3)support subset of SSL (with default configuration)
*/
public class JMXListener implements Coprocessor {
public static final Log LOG = LogFactory.getLog(JMXListener.class);
public static final String RMI_REGISTRY_PORT_CONF_KEY = ".rmi.registry.port";
public static final String RMI_CONNECTOR_PORT_CONF_KEY = ".rmi.connector.port";
public static int defRMIRegistryPort = 10102;
/**
* workaround for HBASE-11146
* master and regionserver are in 1 JVM in standalone mode
* only 1 JMX instance is allowed, otherwise there is port conflict even if
* we only load regionserver coprocessor on master
*/
private static JMXConnectorServer jmxCS = null;
public static JMXServiceURL buildJMXServiceURL(int rmiRegistryPort,
int rmiConnectorPort) throws IOException {
// Build jmxURL
StringBuilder url = new StringBuilder();
url.append("service:jmx:rmi://localhost:");
url.append(rmiConnectorPort);
url.append("/jndi/rmi://localhost:");
url.append(rmiRegistryPort);
url.append("/jmxrmi");
return new JMXServiceURL(url.toString());
}
public void startConnectorServer(int rmiRegistryPort, int rmiConnectorPort)
throws IOException {
boolean rmiSSL = false;
boolean authenticate = true;
String passwordFile = null;
String accessFile = null;
System.setProperty("java.rmi.server.randomIDs", "true");
String rmiSSLValue = System.getProperty("com.sun.management.jmxremote.ssl",
"false");
rmiSSL = Boolean.parseBoolean(rmiSSLValue);
String authenticateValue =
System.getProperty("com.sun.management.jmxremote.authenticate", "false");
authenticate = Boolean.parseBoolean(authenticateValue);
passwordFile = System.getProperty("com.sun.management.jmxremote.password.file");
accessFile = System.getProperty("com.sun.management.jmxremote.access.file");
LOG.info("rmiSSL:" + rmiSSLValue + ",authenticate:" + authenticateValue
+ ",passwordFile:" + passwordFile + ",accessFile:" + accessFile);
// Environment map
HashMap<String, Object> jmxEnv = new HashMap<String, Object>();
RMIClientSocketFactory csf = null;
RMIServerSocketFactory ssf = null;
if (rmiSSL) {
if (rmiRegistryPort == rmiConnectorPort) {
throw new IOException("SSL is enabled. " +
"rmiConnectorPort cannot share with the rmiRegistryPort!");
}
csf = new SslRMIClientSocketFactory();
ssf = new SslRMIServerSocketFactory();
}
if (csf != null) {
jmxEnv.put(RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE, csf);
}
if (ssf != null) {
jmxEnv.put(RMIConnectorServer.RMI_SERVER_SOCKET_FACTORY_ATTRIBUTE, ssf);
}
// Configure authentication
if (authenticate) {
jmxEnv.put("jmx.remote.x.password.file", passwordFile);
jmxEnv.put("jmx.remote.x.access.file", accessFile);
}
// Create the RMI registry
LocateRegistry.createRegistry(rmiRegistryPort);
// Retrieve the PlatformMBeanServer.
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
// Build jmxURL
JMXServiceURL serviceUrl = buildJMXServiceURL(rmiRegistryPort, rmiConnectorPort);
try {
// Start the JMXListener with the connection string
jmxCS = JMXConnectorServerFactory.newJMXConnectorServer(serviceUrl, jmxEnv, mbs);
jmxCS.start();
LOG.info("ConnectorServer started!");
} catch (IOException e) {
LOG.error("fail to start connector server!", e);
}
}
public void stopConnectorServer() throws IOException {
synchronized(JMXListener.class) {
if (jmxCS != null) {
jmxCS.stop();
LOG.info("ConnectorServer stopped!");
jmxCS = null;
}
}
}
@Override
public void start(CoprocessorEnvironment env) throws IOException {
int rmiRegistryPort = -1;
int rmiConnectorPort = -1;
Configuration conf = env.getConfiguration();
if (env instanceof MasterCoprocessorEnvironment) {
LOG.error("JMXListener should not be loaded in Master Environment!");
} else if (env instanceof RegionServerCoprocessorEnvironment) {
// running on RegionServer --since 0.99 HMaster is also a HRegionServer
rmiRegistryPort =
conf.getInt("regionserver" + RMI_REGISTRY_PORT_CONF_KEY, defRMIRegistryPort);
rmiConnectorPort =
conf.getInt("regionserver" + RMI_CONNECTOR_PORT_CONF_KEY, rmiRegistryPort);
LOG.info("RegionServer rmiRegistryPort:" + rmiRegistryPort
+ ",RegionServer rmiConnectorPort:" + rmiConnectorPort);
} else if (env instanceof RegionCoprocessorEnvironment) {
LOG.error("JMXListener should not be loaded in Region Environment!");
}
synchronized(JMXListener.class) {
if (jmxCS != null) {
LOG.info("JMXListener has been started at Registry port " + rmiRegistryPort);
}
else {
startConnectorServer(rmiRegistryPort, rmiConnectorPort);
}
}
}
@Override
public void stop(CoprocessorEnvironment env) throws IOException {
stopConnectorServer();
}
}

95
hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java Normal file
View File

@ -0,0 +1,95 @@
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase;
import java.io.IOException;
import javax.management.MBeanServerConnection;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
@Category(MediumTests.class)
public class TestJMXListener {
private static final Log LOG = LogFactory.getLog(TestJMXListener.class);
private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static int connectorPort = 61120;
@BeforeClass
public static void setupBeforeClass() throws Exception {
Configuration conf = UTIL.getConfiguration();
conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
JMXListener.class.getName());
conf.setInt("regionserver.rmi.registry.port", connectorPort);
UTIL.startMiniCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
UTIL.shutdownMiniCluster();
}
@Test
public void testStart() throws Exception {
JMXConnector connector = JMXConnectorFactory.connect(
JMXListener.buildJMXServiceURL(connectorPort,connectorPort));
MBeanServerConnection mb = connector.getMBeanServerConnection();
String domain = mb.getDefaultDomain();
Assert.assertTrue("default domain is not correct",
!domain.isEmpty());
connector.close();
}
//shutdown hbase only. then try connect, IOException expected
@Rule
public ExpectedException expectedEx = ExpectedException.none();
@Test
public void testStop() throws Exception {
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
LOG.info("shutdown hbase cluster...");
cluster.shutdown();
LOG.info("wait for the hbase cluster shutdown...");
cluster.waitUntilShutDown();
JMXConnector connector = JMXConnectorFactory.newJMXConnector(
JMXListener.buildJMXServiceURL(connectorPort,connectorPort), null);
expectedEx.expect(IOException.class);
connector.connect();
}
}

107
src/main/docbkx/configuration.xml
View File

@ -1411,7 +1411,8 @@ index e70ebc6..96f8c27 100644
a late-version HDFS so you have the fixes he refers too and himself adds to HDFS that help
HBase MTTR (e.g. HDFS-3703, HDFS-3712, and HDFS-4791 -- hadoop 2 for sure has them and
late hadoop 1 has some). Set the following in the RegionServer. </para>
<programlisting><![CDATA[<property>
<programlisting><![CDATA[
<property>
<name>hbase.lease.recovery.dfs.timeout</name>
<value>23000</value>
<description>How much time we allow elapse between calls to recover lease.
@ -1421,12 +1422,13 @@ index e70ebc6..96f8c27 100644
<name>dfs.client.socket-timeout</name>
<value>10000</value>
<description>Down the DFS timeout from 60 to 10 seconds.</description>
</property>]]>
</programlisting>
</property>
]]></programlisting>
<para>And on the namenode/datanode side, set the following to enable 'staleness' introduced
in HDFS-3703, HDFS-3912. </para>
<programlisting><![CDATA[<property>
<programlisting><![CDATA[
<property>
<name>dfs.client.socket-timeout</name>
<value>10000</value>
<description>Down the DFS timeout from 60 to 10 seconds.</description>
@ -1460,12 +1462,105 @@ index e70ebc6..96f8c27 100644
<name>dfs.namenode.avoid.write.stale.datanode</name>
<value>true</value>
<description>Enable stale state in hdfs</description>
</property>]]>
</programlisting>
</property>
]]></programlisting>
</section>
<section
xml:id="JMX_config">
<title>JMX</title>
<para>JMX(Java Management Extensions) provides built-in instrumentation that enables you
to monitor and manage the Java VM. To enable monitoring and management from remote
systems, you need to set system property com.sun.management.jmxremote.port(the port
number through which you want to enable JMX RMI connections) when you start the Java VM.
See <link
xlink:href="http://docs.oracle.com/javase/6/docs/technotes/guides/management/agent.html">
official document</link> for more information. Historically, besides above port mentioned,
JMX opens 2 additional random TCP listening ports, which could lead to port conflict
problem.(See <link
xlink:href="https://issues.apache.org/jira/browse/HBASE-10289">HBASE-10289</link>
for details)
</para>
<para>As an alternative, You can use the coprocessor-based JMX implementation provided
by HBase. To enable it, add below property in <filename>hbase-site.xml</filename>:
<programlisting><![CDATA[
<property>
<name>hbase.coprocessor.regionserver.classes</name>
<value>org.apache.hadoop.hbase.JMXListener</value>
</property>
]]></programlisting>
NOTE: DO NOT set com.sun.management.jmxremote.port for Java VM at the same time.
</para>
<para>Currently it supports Master and RegionServer Java VM. The reason why you only
configure coprocessor for 'regionserver' is that, starting from HBase 0.99,
a Master IS also a RegionServer. (See <link
xlink:href="https://issues.apache.org/jira/browse/HBASE-10569">HBASE-10569</link>
for more information.)
By default, the JMX listens on TCP port 10102, you can further configure the port
using below properties:
<programlisting><![CDATA[
<property>
<name>regionserver.rmi.registry.port</name>
<value>61130</value>
</property>
<property>
<name>regionserver.rmi.connector.port</name>
<value>61140</value>
</property>
]]></programlisting>
The registry port can be shared with connector port in most cases, so you only
need to configure regionserver.rmi.registry.port. However if you want to use SSL
communication, the 2 ports must be configured to different values.
</para>
<para>By default the password authentication and SSL communication is disabled.
To enable password authentication, you need to update <filename>hbase-env.sh</filename>
like below:
<screen>
export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.authenticate=true \
-Dcom.sun.management.jmxremote.password.file=your_password_file \
-Dcom.sun.management.jmxremote.access.file=your_access_file"
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE "
export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE "
</screen>
See example password/access file under $JRE_HOME/lib/management.
</para>
<para>To enable SSL communication with password authentication, follow below steps:
<screen>
#1. generate a key pair, stored in myKeyStore
keytool -genkey -alias jconsole -keystore myKeyStore
#2. export it to file jconsole.cert
keytool -export -alias jconsole -keystore myKeyStore -file jconsole.cert
#3. copy jconsole.cert to jconsole client machine, import it to jconsoleKeyStore
keytool -import -alias jconsole -keystore jconsoleKeyStore -file jconsole.cert
</screen>
And then update <filename>hbase-env.sh</filename> like below:
<screen>
export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=true \
-Djavax.net.ssl.keyStore=/home/tianq/myKeyStore \
-Djavax.net.ssl.keyStorePassword=your_password_in_step_#1 \
-Dcom.sun.management.jmxremote.authenticate=true \
-Dcom.sun.management.jmxremote.password.file=your_password file \
-Dcom.sun.management.jmxremote.access.file=your_access_file"
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE "
export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE "
</screen>
Finally start jconsole on client using the key store:
<screen>
jconsole -J-Djavax.net.ssl.trustStore=/home/tianq/jconsoleKeyStore
</screen>
</para>
</section>
</section>
</section>
<!-- important config -->