HBASE-10289 Avoid random port usage by default JMX Server. Create Custome JMX server (Qiang Tian)

conf/hbase-env.sh

@@ -76,7 +76,9 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
 # Uncomment and adjust to enable JMX exporting
 # See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
 # More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
-#
+# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX
+# section in HBase Reference Guide for instructions.
+
 # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
 # export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
 # export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"

hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java

@@ -0,0 +1,195 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.*;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.rmi.registry.LocateRegistry;
+import java.rmi.server.RMIClientSocketFactory;
+import java.rmi.server.RMIServerSocketFactory;
+import java.util.HashMap;
+
+import javax.management.MBeanServer;
+import javax.management.remote.JMXConnectorServer;
+import javax.management.remote.JMXConnectorServerFactory;
+import javax.management.remote.JMXServiceURL;
+import javax.management.remote.rmi.RMIConnectorServer;
+import javax.rmi.ssl.SslRMIClientSocketFactory;
+import javax.rmi.ssl.SslRMIServerSocketFactory;
+
+/**
+ * Pluggable JMX Agent for HBase(to fix the 2 random TCP ports issue
+ * of the out-of-the-box JMX Agent):
+ * 1)connector port can share with the registry port if SSL is OFF
+ * 2)support password authentication
+ * 3)support subset of SSL (with default configuration)
+ */
+public class JMXListener implements Coprocessor {
+
+  public static final Log LOG = LogFactory.getLog(JMXListener.class);
+  public static final String RMI_REGISTRY_PORT_CONF_KEY = ".rmi.registry.port";
+  public static final String RMI_CONNECTOR_PORT_CONF_KEY = ".rmi.connector.port";
+  public static int defRMIRegistryPort = 10102;
+
+  /**
+   * workaround for HBASE-11146
+   * master and regionserver are in 1 JVM in standalone mode
+   * only 1 JMX instance is allowed, otherwise there is port conflict even if
+   * we only load regionserver coprocessor on master
+   */
+  private static JMXConnectorServer jmxCS = null;
+
+  public static JMXServiceURL buildJMXServiceURL(int rmiRegistryPort,
+      int rmiConnectorPort) throws IOException {
+    // Build jmxURL
+    StringBuilder url = new StringBuilder();
+    url.append("service:jmx:rmi://localhost:");
+    url.append(rmiConnectorPort);
+    url.append("/jndi/rmi://localhost:");
+    url.append(rmiRegistryPort);
+    url.append("/jmxrmi");
+
+    return new JMXServiceURL(url.toString());
+
+  }
+
+  public void startConnectorServer(int rmiRegistryPort, int rmiConnectorPort)
+              throws IOException {
+    boolean rmiSSL = false;
+    boolean authenticate = true;
+    String passwordFile = null;
+    String accessFile = null;
+
+    System.setProperty("java.rmi.server.randomIDs", "true");
+
+    String rmiSSLValue = System.getProperty("com.sun.management.jmxremote.ssl",
+                                            "false");
+    rmiSSL = Boolean.parseBoolean(rmiSSLValue);
+
+    String authenticateValue =
+        System.getProperty("com.sun.management.jmxremote.authenticate", "false");
+    authenticate = Boolean.parseBoolean(authenticateValue);
+
+    passwordFile = System.getProperty("com.sun.management.jmxremote.password.file");
+    accessFile = System.getProperty("com.sun.management.jmxremote.access.file");
+
+    LOG.info("rmiSSL:" + rmiSSLValue + ",authenticate:" + authenticateValue
+              + ",passwordFile:" + passwordFile + ",accessFile:" + accessFile);
+
+    // Environment map
+    HashMap<String, Object> jmxEnv = new HashMap<String, Object>();
+
+    RMIClientSocketFactory csf = null;
+    RMIServerSocketFactory ssf = null;
+
+    if (rmiSSL) {
+      if (rmiRegistryPort == rmiConnectorPort) {
+        throw new IOException("SSL is enabled. " +
+            "rmiConnectorPort cannot share with the rmiRegistryPort!");
+      }
+      csf = new SslRMIClientSocketFactory();
+      ssf = new SslRMIServerSocketFactory();
+    }
+
+    if (csf != null) {
+      jmxEnv.put(RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE, csf);
+    }
+    if (ssf != null) {
+      jmxEnv.put(RMIConnectorServer.RMI_SERVER_SOCKET_FACTORY_ATTRIBUTE, ssf);
+    }
+
+    // Configure authentication
+    if (authenticate) {
+      jmxEnv.put("jmx.remote.x.password.file", passwordFile);
+      jmxEnv.put("jmx.remote.x.access.file", accessFile);
+    }
+
+    // Create the RMI registry
+    LocateRegistry.createRegistry(rmiRegistryPort);
+    // Retrieve the PlatformMBeanServer.
+    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+
+    // Build jmxURL
+    JMXServiceURL serviceUrl = buildJMXServiceURL(rmiRegistryPort, rmiConnectorPort);
+
+    try {
+      // Start the JMXListener with the connection string
+      jmxCS = JMXConnectorServerFactory.newJMXConnectorServer(serviceUrl, jmxEnv, mbs);
+      jmxCS.start();
+      LOG.info("ConnectorServer started!");
+    } catch (IOException e) {
+      LOG.error("fail to start connector server!", e);
+    }
+
+  }
+
+  public void stopConnectorServer() throws IOException {
+    synchronized(JMXListener.class) {
+      if (jmxCS != null) {
+        jmxCS.stop();
+        LOG.info("ConnectorServer stopped!");
+        jmxCS = null;
+      }
+    }
+  }
+
+
+  @Override
+  public void start(CoprocessorEnvironment env) throws IOException {
+    int rmiRegistryPort = -1;
+    int rmiConnectorPort = -1;
+    Configuration conf = env.getConfiguration();
+
+    if (env instanceof MasterCoprocessorEnvironment) {
+      LOG.error("JMXListener should not be loaded in Master Environment!");
+    } else if (env instanceof RegionServerCoprocessorEnvironment) {
+      // running on RegionServer --since 0.99 HMaster is also a HRegionServer
+      rmiRegistryPort =
+        conf.getInt("regionserver" + RMI_REGISTRY_PORT_CONF_KEY, defRMIRegistryPort);
+      rmiConnectorPort =
+        conf.getInt("regionserver" + RMI_CONNECTOR_PORT_CONF_KEY, rmiRegistryPort);
+      LOG.info("RegionServer rmiRegistryPort:" + rmiRegistryPort
+        + ",RegionServer rmiConnectorPort:" + rmiConnectorPort);
+
+    } else if (env instanceof RegionCoprocessorEnvironment) {
+      LOG.error("JMXListener should not be loaded in Region Environment!");
+    }
+
+    synchronized(JMXListener.class) {
+      if (jmxCS != null) {
+        LOG.info("JMXListener has been started at Registry port " + rmiRegistryPort);
+      }
+      else {
+        startConnectorServer(rmiRegistryPort, rmiConnectorPort);
+      }
+    }
+  }
+
+  @Override
+  public void stop(CoprocessorEnvironment env) throws IOException {
+    stopConnectorServer();
+  }
+
+}

hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java

@@ -0,0 +1,95 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import javax.management.MBeanServerConnection;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+
+
+
+@Category(MediumTests.class)
+public class TestJMXListener {
+  private static final Log LOG = LogFactory.getLog(TestJMXListener.class);
+  private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private static int connectorPort = 61120;
+
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+    Configuration conf = UTIL.getConfiguration();
+
+    conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
+      JMXListener.class.getName());
+    conf.setInt("regionserver.rmi.registry.port", connectorPort);
+
+    UTIL.startMiniCluster();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testStart() throws Exception {
+    JMXConnector connector = JMXConnectorFactory.connect(
+      JMXListener.buildJMXServiceURL(connectorPort,connectorPort));
+
+    MBeanServerConnection mb = connector.getMBeanServerConnection();
+    String domain = mb.getDefaultDomain();
+    Assert.assertTrue("default domain is not correct",
+      !domain.isEmpty());
+    connector.close();
+
+  }
+
+  //shutdown hbase only. then try connect, IOException expected
+  @Rule
+  public ExpectedException expectedEx = ExpectedException.none();
+  @Test
+  public void testStop() throws Exception {
+    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
+    LOG.info("shutdown hbase cluster...");
+    cluster.shutdown();
+    LOG.info("wait for the hbase cluster shutdown...");
+    cluster.waitUntilShutDown();
+
+    JMXConnector connector = JMXConnectorFactory.newJMXConnector(
+      JMXListener.buildJMXServiceURL(connectorPort,connectorPort), null);
+    expectedEx.expect(IOException.class);
+    connector.connect();
+
+  }
+
+
+}

src/main/docbkx/configuration.xml

@@ -1411,7 +1411,8 @@ index e70ebc6..96f8c27 100644
           a late-version HDFS so you have the fixes he refers too and himself adds to HDFS that help
           HBase MTTR (e.g. HDFS-3703, HDFS-3712, and HDFS-4791 -- hadoop 2 for sure has them and
           late hadoop 1 has some). Set the following in the RegionServer. </para>
-        <programlisting><![CDATA[<property>
+        <programlisting><![CDATA[
+<property>
     <name>hbase.lease.recovery.dfs.timeout</name>
     <value>23000</value>
     <description>How much time we allow elapse between calls to recover lease.
@@ -1421,12 +1422,13 @@ index e70ebc6..96f8c27 100644
     <name>dfs.client.socket-timeout</name>
     <value>10000</value>
     <description>Down the DFS timeout from 60 to 10 seconds.</description>
-</property>]]>
+</property>
-        </programlisting>
+]]></programlisting>
 
         <para>And on the namenode/datanode side, set the following to enable 'staleness' introduced
           in HDFS-3703, HDFS-3912. </para>
-        <programlisting><![CDATA[<property>
+        <programlisting><![CDATA[
+<property>
     <name>dfs.client.socket-timeout</name>
     <value>10000</value>
     <description>Down the DFS timeout from 60 to 10 seconds.</description>
@@ -1460,12 +1462,105 @@ index e70ebc6..96f8c27 100644
     <name>dfs.namenode.avoid.write.stale.datanode</name>
     <value>true</value>
     <description>Enable stale state in hdfs</description>
-</property>]]>
+</property>
-        </programlisting>
+]]></programlisting>
       </section>
 
+      <section
+        xml:id="JMX_config">
+        <title>JMX</title>
+        <para>JMX(Java Management Extensions) provides built-in instrumentation that enables you
+          to monitor and manage the Java VM. To enable monitoring and management from remote
+          systems, you need to set system property com.sun.management.jmxremote.port(the port
+          number through which you want to enable JMX RMI connections) when you start the Java VM.
+          See <link
+            xlink:href="http://docs.oracle.com/javase/6/docs/technotes/guides/management/agent.html">
+          official document</link> for more information. Historically, besides above port mentioned,
+          JMX opens 2 additional random TCP listening ports, which could lead to port conflict
+          problem.(See <link
+            xlink:href="https://issues.apache.org/jira/browse/HBASE-10289">HBASE-10289</link>
+          for details)
+        </para>
+        <para>As an alternative, You can use the coprocessor-based JMX implementation provided
+          by HBase. To enable it, add below property in <filename>hbase-site.xml</filename>:
+        <programlisting><![CDATA[
+<property>
+    <name>hbase.coprocessor.regionserver.classes</name>
+    <value>org.apache.hadoop.hbase.JMXListener</value>
+</property>
+]]></programlisting>
+          NOTE: DO NOT set com.sun.management.jmxremote.port for Java VM at the same time.
+        </para>
+        <para>Currently it supports Master and RegionServer Java VM. The reason why you only
+          configure coprocessor for 'regionserver' is that, starting from HBase 0.99,
+          a Master IS also a RegionServer. (See <link
+            xlink:href="https://issues.apache.org/jira/browse/HBASE-10569">HBASE-10569</link>
+          for more information.)
+          By default, the JMX listens on TCP port 10102, you can further configure the port
+          using below properties:
+
+        <programlisting><![CDATA[
+<property>
+    <name>regionserver.rmi.registry.port</name>
+    <value>61130</value>
+</property>
+<property>
+    <name>regionserver.rmi.connector.port</name>
+    <value>61140</value>
+</property>
+]]></programlisting>
+          The registry port can be shared with connector port in most cases, so you only
+          need to configure regionserver.rmi.registry.port. However if you want to use SSL
+          communication, the 2 ports must be configured to different values.
+        </para>
+
+        <para>By default the password authentication and SSL communication is disabled.
+          To enable password authentication, you need to update <filename>hbase-env.sh</filename>
+          like below:
+      <screen>
+export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.authenticate=true                  \
+                       -Dcom.sun.management.jmxremote.password.file=your_password_file   \
+                       -Dcom.sun.management.jmxremote.access.file=your_access_file"
+
+export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE "
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE "
+      </screen>
+          See example password/access file under $JRE_HOME/lib/management.
+        </para>
+
+        <para>To enable SSL communication with password authentication, follow below steps:
+      <screen>
+#1. generate a key pair, stored in myKeyStore
+keytool -genkey -alias jconsole -keystore myKeyStore
+
+#2. export it to file jconsole.cert
+keytool -export -alias jconsole -keystore myKeyStore -file jconsole.cert
+
+#3. copy jconsole.cert to jconsole client machine, import it to jconsoleKeyStore
+keytool -import -alias jconsole -keystore jconsoleKeyStore -file jconsole.cert
+      </screen>
+          And then update <filename>hbase-env.sh</filename> like below:
+      <screen>
+export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=true                         \
+                       -Djavax.net.ssl.keyStore=/home/tianq/myKeyStore                 \
+                       -Djavax.net.ssl.keyStorePassword=your_password_in_step_#1       \
+                       -Dcom.sun.management.jmxremote.authenticate=true                \
+                       -Dcom.sun.management.jmxremote.password.file=your_password file \
+                       -Dcom.sun.management.jmxremote.access.file=your_access_file"
+
+export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE "
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE "
+      </screen>
+
+          Finally start jconsole on client using the key store:
+      <screen>
+jconsole -J-Djavax.net.ssl.trustStore=/home/tianq/jconsoleKeyStore
+      </screen>
+        </para>
     </section>
 
+   </section>
+
   </section>
   <!--  important config -->