From 61f96b6ffad49a4db44de2366fb6e18d1053b604 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Wed, 9 May 2018 23:23:29 -0700 Subject: [PATCH] HBASE-20544 Make HBTU default to random ports. Signed-off-by: Umesh Agashe Signed-off-by: Josh Elser Conflicts: hbase-backup/src/test/resources/hbase-site.xml hbase-spark-it/src/test/resources/hbase-site.xml hbase-spark/src/test/resources/hbase-site.xml --- .../src/test/resources/hbase-site.xml | 7 -- .../src/test/resources/hbase-site.xml | 7 -- .../src/test/resources/hbase-site.xml | 7 -- .../src/test/resources/hbase-site.xml | 7 -- .../src/test/resources/hbase-site.xml | 7 -- .../src/test/resources/hbase-site.xml | 7 -- .../src/test/resources/hbase-site.xml | 7 -- hbase-rest/src/test/resources/hbase-site.xml | 7 -- .../hbase/rsgroup/TestEnableRSGroup.java | 4 +- .../src/test/resources/hbase-site.xml | 7 -- .../hadoop/hbase/LocalHBaseCluster.java | 27 ++++++-- .../hadoop/hbase/HBaseTestingUtility.java | 18 ++--- .../apache/hadoop/hbase/MiniHBaseCluster.java | 6 -- .../hadoop/hbase/TestHBaseTestingUtility.java | 33 ++++++--- .../src/test/resources/hbase-site.xml | 7 -- hbase-shell/src/test/resources/hbase-site.xml | 7 -- .../hbase/TestHBaseTestingUtilSpinup.java | 67 +++++++++++++++++++ .../src/test/resources/hbase-site.xml | 7 -- 18 files changed, 117 insertions(+), 122 deletions(-) create mode 100644 hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java diff --git a/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml b/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml index 858d4285cf0..99d2ab8d1fb 100644 --- a/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml +++ b/hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml b/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml index 858d4285cf0..99d2ab8d1fb 100644 --- a/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml +++ b/hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-client/src/test/resources/hbase-site.xml b/hbase-client/src/test/resources/hbase-site.xml index 858d4285cf0..99d2ab8d1fb 100644 --- a/hbase-client/src/test/resources/hbase-site.xml +++ b/hbase-client/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-endpoint/src/test/resources/hbase-site.xml b/hbase-endpoint/src/test/resources/hbase-site.xml index 858d4285cf0..99d2ab8d1fb 100644 --- a/hbase-endpoint/src/test/resources/hbase-site.xml +++ b/hbase-endpoint/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-examples/src/test/resources/hbase-site.xml b/hbase-examples/src/test/resources/hbase-site.xml index 697567e9f35..ab4d1cd0b73 100644 --- a/hbase-examples/src/test/resources/hbase-site.xml +++ b/hbase-examples/src/test/resources/hbase-site.xml @@ -25,11 +25,4 @@ hbase.defaults.for.version.skip true - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-mapreduce/src/test/resources/hbase-site.xml b/hbase-mapreduce/src/test/resources/hbase-site.xml index 34802d0e9c1..64a19644358 100644 --- a/hbase-mapreduce/src/test/resources/hbase-site.xml +++ b/hbase-mapreduce/src/test/resources/hbase-site.xml @@ -158,11 +158,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml index a1cc27ebf16..114ee8a23c1 100644 --- a/hbase-procedure/src/test/resources/hbase-site.xml +++ b/hbase-procedure/src/test/resources/hbase-site.xml @@ -41,11 +41,4 @@ WARNING: Doing so may expose you to additional risk of data loss! - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-rest/src/test/resources/hbase-site.xml b/hbase-rest/src/test/resources/hbase-site.xml index be7b49243a4..2bd3ee45a42 100644 --- a/hbase-rest/src/test/resources/hbase-site.xml +++ b/hbase-rest/src/test/resources/hbase-site.xml @@ -139,11 +139,4 @@ Skip sanity checks in tests - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java index 1407a5b94f2..04d9d08a8db 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java @@ -49,10 +49,10 @@ public class TestEnableRSGroup { protected static final Logger LOG = LoggerFactory.getLogger(TestEnableRSGroup.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static Configuration conf = TEST_UTIL.getConfiguration(); @BeforeClass public static void setUp() throws Exception { + final Configuration conf = TEST_UTIL.getConfiguration(); conf.setBoolean(CoprocessorHost.COPROCESSORS_ENABLED_CONF_KEY, true); TEST_UTIL.startMiniCluster(5); } @@ -69,9 +69,9 @@ public class TestEnableRSGroup { TEST_UTIL.getMiniHBaseCluster().waitOnMaster(0); LOG.info("stopped master..."); + final Configuration conf = TEST_UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, RSGroupAdminEndpoint.class.getName()); conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); - TEST_UTIL.getMiniHBaseCluster().setConf(conf); TEST_UTIL.getMiniHBaseCluster().startMaster(); TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(60000); diff --git a/hbase-rsgroup/src/test/resources/hbase-site.xml b/hbase-rsgroup/src/test/resources/hbase-site.xml index 858d4285cf0..99d2ab8d1fb 100644 --- a/hbase-rsgroup/src/test/resources/hbase-site.xml +++ b/hbase-rsgroup/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java index 5c8ddd9ba64..19143932b5a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java @@ -139,15 +139,30 @@ public class LocalHBaseCluster { throws IOException { this.conf = conf; - // Always have masters and regionservers come up on port '0' so we don't - // clash over default ports. + // When active, if a port selection is default then we switch to random if (conf.getBoolean(ASSIGN_RANDOM_PORTS, false)) { - conf.set(HConstants.MASTER_PORT, "0"); - conf.set(HConstants.REGIONSERVER_PORT, "0"); - if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) { + if (conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT) + == HConstants.DEFAULT_MASTER_PORT) { + LOG.debug("Setting Master Port to random."); + conf.set(HConstants.MASTER_PORT, "0"); + } + if (conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT) + == HConstants.DEFAULT_REGIONSERVER_PORT) { + LOG.debug("Setting RegionServer Port to random."); + conf.set(HConstants.REGIONSERVER_PORT, "0"); + } + // treat info ports special; expressly don't change '-1' (keep off) + // in case we make that the default behavior. + if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1 && + conf.getInt(HConstants.REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT) + == HConstants.DEFAULT_REGIONSERVER_INFOPORT) { + LOG.debug("Setting RS InfoServer Port to random."); conf.set(HConstants.REGIONSERVER_INFO_PORT, "0"); } - if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) { + if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1 && + conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT) + == HConstants.DEFAULT_MASTER_INFOPORT) { + LOG.debug("Setting Master InfoServer Port to random."); conf.set(HConstants.MASTER_INFO_PORT, "0"); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 8faeb5d293c..ba40fbf272b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -333,20 +333,6 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { // a hbase checksum verification failure will cause unit tests to fail ChecksumUtil.generateExceptionForChecksumFailureForTest(true); - // if conf is provided, prevent contention for ports if other hbase thread(s) are running - if (conf != null) { - if (conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT) - == HConstants.DEFAULT_MASTER_INFOPORT) { - conf.setInt(HConstants.MASTER_INFO_PORT, -1); - LOG.debug("Config property {} changed to -1", HConstants.MASTER_INFO_PORT); - } - if (conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT) - == HConstants.DEFAULT_REGIONSERVER_PORT) { - conf.setInt(HConstants.REGIONSERVER_PORT, -1); - LOG.debug("Config property {} changed to -1", HConstants.REGIONSERVER_PORT); - } - } - // Save this for when setting default file:// breaks things this.conf.set("original.defaultFS", this.conf.get("fs.defaultFS")); @@ -357,6 +343,10 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE,false); + // If the value for random ports isn't set set it to true, thus making + // tests opt-out for random port assignment + this.conf.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, + this.conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true)); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index 7b65371fce0..5eb72188e90 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -108,12 +108,6 @@ public class MiniHBaseCluster extends HBaseCluster { Class regionserverClass) throws IOException, InterruptedException { super(conf); - if (conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, false)) { - conf.set(HConstants.MASTER_PORT, "0"); - if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) { - conf.set(HConstants.MASTER_INFO_PORT, "0"); - } - } // Hadoop 2 CompatibilityFactory.getInstance(MetricsAssertHelper.class).init(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java index f49171c5182..97de8a9e773 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java @@ -436,26 +436,39 @@ public class TestHBaseTestingUtility { } @Test - public void testOverridingOfDefaultPorts() { + public void testOverridingOfDefaultPorts() throws Exception { - // confirm that default port properties being overridden to "-1" + // confirm that default port properties being overridden to random Configuration defaultConfig = HBaseConfiguration.create(); defaultConfig.setInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT); - defaultConfig.setInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT); + defaultConfig.setInt(HConstants.REGIONSERVER_INFO_PORT, + HConstants.DEFAULT_REGIONSERVER_INFOPORT); HBaseTestingUtility htu = new HBaseTestingUtility(defaultConfig); - assertEquals(-1, htu.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0)); - assertEquals(-1, htu.getConfiguration().getInt(HConstants.REGIONSERVER_PORT, 0)); + try { + MiniHBaseCluster defaultCluster = htu.startMiniCluster(); + assertNotEquals(HConstants.DEFAULT_MASTER_INFOPORT, + defaultCluster.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0)); + assertNotEquals(HConstants.DEFAULT_REGIONSERVER_INFOPORT, + defaultCluster.getConfiguration().getInt(HConstants.REGIONSERVER_INFO_PORT, 0)); + } finally { + htu.shutdownMiniCluster(); + } // confirm that nonDefault (custom) port settings are NOT overridden Configuration altConfig = HBaseConfiguration.create(); final int nonDefaultMasterInfoPort = 3333; final int nonDefaultRegionServerPort = 4444; altConfig.setInt(HConstants.MASTER_INFO_PORT, nonDefaultMasterInfoPort); - altConfig.setInt(HConstants.REGIONSERVER_PORT, nonDefaultRegionServerPort); + altConfig.setInt(HConstants.REGIONSERVER_INFO_PORT, nonDefaultRegionServerPort); htu = new HBaseTestingUtility(altConfig); - assertEquals(nonDefaultMasterInfoPort, - htu.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0)); - assertEquals(nonDefaultRegionServerPort - , htu.getConfiguration().getInt(HConstants.REGIONSERVER_PORT, 0)); + try { + MiniHBaseCluster customCluster = htu.startMiniCluster(); + assertEquals(nonDefaultMasterInfoPort, + customCluster.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0)); + assertEquals(nonDefaultRegionServerPort, + customCluster.getConfiguration().getInt(HConstants.REGIONSERVER_INFO_PORT, 0)); + } finally { + htu.shutdownMiniCluster(); + } } } diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index 34802d0e9c1..64a19644358 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -158,11 +158,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-shell/src/test/resources/hbase-site.xml b/hbase-shell/src/test/resources/hbase-site.xml index 858d4285cf0..99d2ab8d1fb 100644 --- a/hbase-shell/src/test/resources/hbase-site.xml +++ b/hbase-shell/src/test/resources/hbase-site.xml @@ -29,11 +29,4 @@ hbase.hconnection.threads.keepalivetime 3 - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - - diff --git a/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java new file mode 100644 index 00000000000..4ea4fc54cff --- /dev/null +++ b/hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertFalse; + +import java.util.List; + +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Make sure we can spin up a HBTU without a hbase-site.xml + */ +@Category({MiscTests.class, MediumTests.class}) +public class TestHBaseTestingUtilSpinup { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestHBaseTestingUtilSpinup.class); + + private static final Logger LOG = LoggerFactory.getLogger(TestHBaseTestingUtilSpinup.class); + private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + @BeforeClass + public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + if (!UTIL.getHBaseCluster().waitForActiveAndReadyMaster(30000)) { + throw new RuntimeException("Active master not ready"); + } + } + + @AfterClass + public static void afterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test + public void testGetMetaTableRows() throws Exception { + List results = UTIL.getMetaTableRows(); + assertFalse("results should have some entries and is empty.", results.isEmpty()); + } + +} diff --git a/hbase-thrift/src/test/resources/hbase-site.xml b/hbase-thrift/src/test/resources/hbase-site.xml index b354a2b3867..b3fb0d90c50 100644 --- a/hbase-thrift/src/test/resources/hbase-site.xml +++ b/hbase-thrift/src/test/resources/hbase-site.xml @@ -154,11 +154,4 @@ Enable replay sanity checks on procedure tests. - - hbase.localcluster.assign.random.ports - true - - Assign random ports to master and RS info server (UI). - -