HBASE-20544 Make HBTU default to random ports.

Signed-off-by: Umesh Agashe <uagashe@cloudera.com>
Signed-off-by: Josh Elser <elserj@apache.org>

 Conflicts:
	hbase-backup/src/test/resources/hbase-site.xml
	hbase-spark-it/src/test/resources/hbase-site.xml
	hbase-spark/src/test/resources/hbase-site.xml

hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml

@@ -29,11 +29,4 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml

@@ -29,11 +29,4 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-client/src/test/resources/hbase-site.xml

@@ -29,11 +29,4 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-endpoint/src/test/resources/hbase-site.xml

@@ -29,11 +29,4 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-examples/src/test/resources/hbase-site.xml

@@ -25,11 +25,4 @@
     <name>hbase.defaults.for.version.skip</name>
     <value>true</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-mapreduce/src/test/resources/hbase-site.xml

@@ -158,11 +158,4 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-procedure/src/test/resources/hbase-site.xml

@@ -41,11 +41,4 @@
       WARNING: Doing so may expose you to additional risk of data loss!
     </description>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-rest/src/test/resources/hbase-site.xml

@@ -139,11 +139,4 @@
     <description>Skip sanity checks in tests
     </description>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java

@@ -49,10 +49,10 @@ public class TestEnableRSGroup {
   protected static final Logger LOG = LoggerFactory.getLogger(TestEnableRSGroup.class);
 
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private static Configuration conf = TEST_UTIL.getConfiguration();
 
   @BeforeClass
   public static void setUp() throws Exception {
+    final Configuration conf = TEST_UTIL.getConfiguration();
     conf.setBoolean(CoprocessorHost.COPROCESSORS_ENABLED_CONF_KEY, true);
     TEST_UTIL.startMiniCluster(5);
   }
@@ -69,9 +69,9 @@ public class TestEnableRSGroup {
     TEST_UTIL.getMiniHBaseCluster().waitOnMaster(0);
 
     LOG.info("stopped master...");
+    final Configuration conf = TEST_UTIL.getConfiguration();
     conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, RSGroupAdminEndpoint.class.getName());
     conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName());
-    TEST_UTIL.getMiniHBaseCluster().setConf(conf);
 
     TEST_UTIL.getMiniHBaseCluster().startMaster();
     TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(60000);

hbase-rsgroup/src/test/resources/hbase-site.xml

@@ -29,11 +29,4 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java

@@ -139,15 +139,30 @@ public class LocalHBaseCluster {
   throws IOException {
     this.conf = conf;
 
-    // Always have masters and regionservers come up on port '0' so we don't
+    // When active, if a port selection is default then we switch to random
-    // clash over default ports.
     if (conf.getBoolean(ASSIGN_RANDOM_PORTS, false)) {
+      if (conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT)
+          == HConstants.DEFAULT_MASTER_PORT) {
+        LOG.debug("Setting Master Port to random.");
         conf.set(HConstants.MASTER_PORT, "0");
+      }
+      if (conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT)
+          == HConstants.DEFAULT_REGIONSERVER_PORT) {
+        LOG.debug("Setting RegionServer Port to random.");
         conf.set(HConstants.REGIONSERVER_PORT, "0");
-      if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) {
+      }
+      // treat info ports special; expressly don't change '-1' (keep off)
+      // in case we make that the default behavior.
+      if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1 &&
+          conf.getInt(HConstants.REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT)
+          == HConstants.DEFAULT_REGIONSERVER_INFOPORT) {
+        LOG.debug("Setting RS InfoServer Port to random.");
         conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
       }
-      if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
+      if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1 &&
+          conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT)
+          == HConstants.DEFAULT_MASTER_INFOPORT) {
+        LOG.debug("Setting Master InfoServer Port to random.");
         conf.set(HConstants.MASTER_INFO_PORT, "0");
       }
     }

hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java

@@ -333,20 +333,6 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
     // a hbase checksum verification failure will cause unit tests to fail
     ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
 
-    // if conf is provided, prevent contention for ports if other hbase thread(s) are running
-    if (conf != null) {
-      if (conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT)
-              == HConstants.DEFAULT_MASTER_INFOPORT) {
-        conf.setInt(HConstants.MASTER_INFO_PORT, -1);
-        LOG.debug("Config property {} changed to -1", HConstants.MASTER_INFO_PORT);
-      }
-      if (conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT)
-              == HConstants.DEFAULT_REGIONSERVER_PORT) {
-        conf.setInt(HConstants.REGIONSERVER_PORT, -1);
-        LOG.debug("Config property {} changed to -1", HConstants.REGIONSERVER_PORT);
-      }
-    }
-
     // Save this for when setting default file:// breaks things
     this.conf.set("original.defaultFS", this.conf.get("fs.defaultFS"));
 
@@ -357,6 +343,10 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
     this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir);
     LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir);
     this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE,false);
+    // If the value for random ports isn't set set it to true, thus making
+    // tests opt-out for random port assignment
+    this.conf.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS,
+        this.conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true));
   }
 
   /**

hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java

@@ -108,12 +108,6 @@ public class MiniHBaseCluster extends HBaseCluster {
          Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
       throws IOException, InterruptedException {
     super(conf);
-    if (conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, false)) {
-      conf.set(HConstants.MASTER_PORT, "0");
-      if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
-        conf.set(HConstants.MASTER_INFO_PORT, "0");
-      }
-    }
 
     // Hadoop 2
     CompatibilityFactory.getInstance(MetricsAssertHelper.class).init();

hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java

@@ -436,26 +436,39 @@ public class TestHBaseTestingUtility {
   }
 
   @Test
-  public void testOverridingOfDefaultPorts() {
+  public void testOverridingOfDefaultPorts() throws Exception {
 
-    // confirm that default port properties being overridden to "-1"
+    // confirm that default port properties being overridden to random
     Configuration defaultConfig = HBaseConfiguration.create();
     defaultConfig.setInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT);
-    defaultConfig.setInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT);
+    defaultConfig.setInt(HConstants.REGIONSERVER_INFO_PORT,
+        HConstants.DEFAULT_REGIONSERVER_INFOPORT);
     HBaseTestingUtility htu = new HBaseTestingUtility(defaultConfig);
-    assertEquals(-1, htu.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0));
+    try {
-    assertEquals(-1, htu.getConfiguration().getInt(HConstants.REGIONSERVER_PORT, 0));
+      MiniHBaseCluster defaultCluster = htu.startMiniCluster();
+      assertNotEquals(HConstants.DEFAULT_MASTER_INFOPORT,
+          defaultCluster.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0));
+      assertNotEquals(HConstants.DEFAULT_REGIONSERVER_INFOPORT,
+          defaultCluster.getConfiguration().getInt(HConstants.REGIONSERVER_INFO_PORT, 0));
+    } finally {
+      htu.shutdownMiniCluster();
+    }
 
     // confirm that nonDefault (custom) port settings are NOT overridden
     Configuration altConfig = HBaseConfiguration.create();
     final int nonDefaultMasterInfoPort = 3333;
     final int nonDefaultRegionServerPort = 4444;
     altConfig.setInt(HConstants.MASTER_INFO_PORT, nonDefaultMasterInfoPort);
-    altConfig.setInt(HConstants.REGIONSERVER_PORT, nonDefaultRegionServerPort);
+    altConfig.setInt(HConstants.REGIONSERVER_INFO_PORT, nonDefaultRegionServerPort);
     htu = new HBaseTestingUtility(altConfig);
+    try {
+      MiniHBaseCluster customCluster = htu.startMiniCluster();
       assertEquals(nonDefaultMasterInfoPort,
-            htu.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0));
+              customCluster.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0));
-    assertEquals(nonDefaultRegionServerPort
+      assertEquals(nonDefaultRegionServerPort,
-            , htu.getConfiguration().getInt(HConstants.REGIONSERVER_PORT, 0));
+          customCluster.getConfiguration().getInt(HConstants.REGIONSERVER_INFO_PORT, 0));
+    } finally {
+      htu.shutdownMiniCluster();
+    }
   }
 }

hbase-server/src/test/resources/hbase-site.xml

@@ -158,11 +158,4 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-shell/src/test/resources/hbase-site.xml

@@ -29,11 +29,4 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>

hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java

@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertFalse;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Make sure we can spin up a HBTU without a hbase-site.xml
+ */
+@Category({MiscTests.class, MediumTests.class})
+public class TestHBaseTestingUtilSpinup {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+      HBaseClassTestRule.forClass(TestHBaseTestingUtilSpinup.class);
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestHBaseTestingUtilSpinup.class);
+  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    UTIL.startMiniCluster();
+    if (!UTIL.getHBaseCluster().waitForActiveAndReadyMaster(30000)) {
+      throw new RuntimeException("Active master not ready");
+    }
+  }
+
+  @AfterClass
+  public static void afterClass() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testGetMetaTableRows() throws Exception {
+    List<byte[]> results = UTIL.getMetaTableRows();
+    assertFalse("results should have some entries and is empty.", results.isEmpty());
+  }
+
+}

hbase-thrift/src/test/resources/hbase-site.xml

@@ -154,11 +154,4 @@
       Enable replay sanity checks on procedure tests.
     </description>
   </property>
-  <property>
-    <name>hbase.localcluster.assign.random.ports</name>
-    <value>true</value>
-    <description>
-      Assign random ports to master and RS info server (UI).
-    </description>
-  </property>
 </configuration>