Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20544 Make HBTU default to random ports. 

Signed-off-by: Umesh Agashe <uagashe@cloudera.com>
Signed-off-by: Josh Elser <elserj@apache.org>

 Conflicts:
	hbase-backup/src/test/resources/hbase-site.xml
	hbase-spark-it/src/test/resources/hbase-site.xml
	hbase-spark/src/test/resources/hbase-site.xml
This commit is contained in:
Sean Busbey 2018-05-09 23:23:29 -07:00
parent c430016cf9
commit 61f96b6ffa
18 changed files with 117 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-archetypes/hbase-client-project/src/test/resources/hbase-site.xml
View File

@ -29,11 +29,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

7
hbase-archetypes/hbase-shaded-client-project/src/test/resources/hbase-site.xml
View File

@ -29,11 +29,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

7
hbase-client/src/test/resources/hbase-site.xml
View File

@ -29,11 +29,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

7
hbase-endpoint/src/test/resources/hbase-site.xml
View File

@ -29,11 +29,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

7
hbase-examples/src/test/resources/hbase-site.xml
View File

@ -25,11 +25,4 @@
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

7
hbase-mapreduce/src/test/resources/hbase-site.xml
View File

@ -158,11 +158,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

7
hbase-procedure/src/test/resources/hbase-site.xml
View File

@ -41,11 +41,4 @@
WARNING: Doing so may expose you to additional risk of data loss!
</description>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

7
hbase-rest/src/test/resources/hbase-site.xml
View File

@ -139,11 +139,4 @@
<description>Skip sanity checks in tests
</description>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

4
hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroup.java
View File

@ -49,10 +49,10 @@ public class TestEnableRSGroup {
protected static final Logger LOG = LoggerFactory.getLogger(TestEnableRSGroup.class);
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static Configuration conf = TEST_UTIL.getConfiguration();
@BeforeClass
public static void setUp() throws Exception {
final Configuration conf = TEST_UTIL.getConfiguration();
conf.setBoolean(CoprocessorHost.COPROCESSORS_ENABLED_CONF_KEY, true);
TEST_UTIL.startMiniCluster(5);
}
@ -69,9 +69,9 @@ public class TestEnableRSGroup {
TEST_UTIL.getMiniHBaseCluster().waitOnMaster(0);
LOG.info("stopped master...");
final Configuration conf = TEST_UTIL.getConfiguration();
conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, RSGroupAdminEndpoint.class.getName());
conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName());
TEST_UTIL.getMiniHBaseCluster().setConf(conf);
TEST_UTIL.getMiniHBaseCluster().startMaster();
TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(60000);

7
hbase-rsgroup/src/test/resources/hbase-site.xml
View File

@ -29,11 +29,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

27
hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
View File

@ -139,15 +139,30 @@ public class LocalHBaseCluster {
throws IOException {
this.conf = conf;
// Always have masters and regionservers come up on port '0' so we don't
// clash over default ports.
// When active, if a port selection is default then we switch to random
if (conf.getBoolean(ASSIGN_RANDOM_PORTS, false)) {
conf.set(HConstants.MASTER_PORT, "0");
conf.set(HConstants.REGIONSERVER_PORT, "0");
if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) {
if (conf.getInt(HConstants.MASTER_PORT, HConstants.DEFAULT_MASTER_PORT)
== HConstants.DEFAULT_MASTER_PORT) {
LOG.debug("Setting Master Port to random.");
conf.set(HConstants.MASTER_PORT, "0");
}
if (conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT)
== HConstants.DEFAULT_REGIONSERVER_PORT) {
LOG.debug("Setting RegionServer Port to random.");
conf.set(HConstants.REGIONSERVER_PORT, "0");
}
// treat info ports special; expressly don't change '-1' (keep off)
// in case we make that the default behavior.
if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1 &&
conf.getInt(HConstants.REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT)
== HConstants.DEFAULT_REGIONSERVER_INFOPORT) {
LOG.debug("Setting RS InfoServer Port to random.");
conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
}
if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1 &&
conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT)
== HConstants.DEFAULT_MASTER_INFOPORT) {
LOG.debug("Setting Master InfoServer Port to random.");
conf.set(HConstants.MASTER_INFO_PORT, "0");
}
}

18
hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
View File

@ -333,20 +333,6 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
// a hbase checksum verification failure will cause unit tests to fail
ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
// if conf is provided, prevent contention for ports if other hbase thread(s) are running
if (conf != null) {
if (conf.getInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT)
== HConstants.DEFAULT_MASTER_INFOPORT) {
conf.setInt(HConstants.MASTER_INFO_PORT, -1);
LOG.debug("Config property {} changed to -1", HConstants.MASTER_INFO_PORT);
}
if (conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT)
== HConstants.DEFAULT_REGIONSERVER_PORT) {
conf.setInt(HConstants.REGIONSERVER_PORT, -1);
LOG.debug("Config property {} changed to -1", HConstants.REGIONSERVER_PORT);
}
}
// Save this for when setting default file:// breaks things
this.conf.set("original.defaultFS", this.conf.get("fs.defaultFS"));
@ -357,6 +343,10 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir);
LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir);
this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE,false);
// If the value for random ports isn't set set it to true, thus making
// tests opt-out for random port assignment
this.conf.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS,
this.conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true));
}
/**

6
hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
View File

@ -108,12 +108,6 @@ public class MiniHBaseCluster extends HBaseCluster {
Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
throws IOException, InterruptedException {
super(conf);
if (conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, false)) {
conf.set(HConstants.MASTER_PORT, "0");
if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
conf.set(HConstants.MASTER_INFO_PORT, "0");
}
}
// Hadoop 2
CompatibilityFactory.getInstance(MetricsAssertHelper.class).init();

33
hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java
View File

@ -436,26 +436,39 @@ public class TestHBaseTestingUtility {
}
@Test
public void testOverridingOfDefaultPorts() {
public void testOverridingOfDefaultPorts() throws Exception {
// confirm that default port properties being overridden to "-1"
// confirm that default port properties being overridden to random
Configuration defaultConfig = HBaseConfiguration.create();
defaultConfig.setInt(HConstants.MASTER_INFO_PORT, HConstants.DEFAULT_MASTER_INFOPORT);
defaultConfig.setInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT);
defaultConfig.setInt(HConstants.REGIONSERVER_INFO_PORT,
HConstants.DEFAULT_REGIONSERVER_INFOPORT);
HBaseTestingUtility htu = new HBaseTestingUtility(defaultConfig);
assertEquals(-1, htu.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0));
assertEquals(-1, htu.getConfiguration().getInt(HConstants.REGIONSERVER_PORT, 0));
try {
MiniHBaseCluster defaultCluster = htu.startMiniCluster();
assertNotEquals(HConstants.DEFAULT_MASTER_INFOPORT,
defaultCluster.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0));
assertNotEquals(HConstants.DEFAULT_REGIONSERVER_INFOPORT,
defaultCluster.getConfiguration().getInt(HConstants.REGIONSERVER_INFO_PORT, 0));
} finally {
htu.shutdownMiniCluster();
}
// confirm that nonDefault (custom) port settings are NOT overridden
Configuration altConfig = HBaseConfiguration.create();
final int nonDefaultMasterInfoPort = 3333;
final int nonDefaultRegionServerPort = 4444;
altConfig.setInt(HConstants.MASTER_INFO_PORT, nonDefaultMasterInfoPort);
altConfig.setInt(HConstants.REGIONSERVER_PORT, nonDefaultRegionServerPort);
altConfig.setInt(HConstants.REGIONSERVER_INFO_PORT, nonDefaultRegionServerPort);
htu = new HBaseTestingUtility(altConfig);
assertEquals(nonDefaultMasterInfoPort,
htu.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0));
assertEquals(nonDefaultRegionServerPort
, htu.getConfiguration().getInt(HConstants.REGIONSERVER_PORT, 0));
try {
MiniHBaseCluster customCluster = htu.startMiniCluster();
assertEquals(nonDefaultMasterInfoPort,
customCluster.getConfiguration().getInt(HConstants.MASTER_INFO_PORT, 0));
assertEquals(nonDefaultRegionServerPort,
customCluster.getConfiguration().getInt(HConstants.REGIONSERVER_INFO_PORT, 0));
} finally {
htu.shutdownMiniCluster();
}
}
}

7
hbase-server/src/test/resources/hbase-site.xml
View File

@ -158,11 +158,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

7
hbase-shell/src/test/resources/hbase-site.xml
View File

@ -29,11 +29,4 @@
<name>hbase.hconnection.threads.keepalivetime</name>
<value>3</value>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>

67
hbase-testing-util/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtilSpinup.java Normal file
View File

@ -0,0 +1,67 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase;
import static org.junit.Assert.assertFalse;
import java.util.List;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Make sure we can spin up a HBTU without a hbase-site.xml
*/
@Category({MiscTests.class, MediumTests.class})
public class TestHBaseTestingUtilSpinup {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestHBaseTestingUtilSpinup.class);
private static final Logger LOG = LoggerFactory.getLogger(TestHBaseTestingUtilSpinup.class);
private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
if (!UTIL.getHBaseCluster().waitForActiveAndReadyMaster(30000)) {
throw new RuntimeException("Active master not ready");
}
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniCluster();
}
@Test
public void testGetMetaTableRows() throws Exception {
List<byte[]> results = UTIL.getMetaTableRows();
assertFalse("results should have some entries and is empty.", results.isEmpty());
}
}

7
hbase-thrift/src/test/resources/hbase-site.xml
View File

@ -154,11 +154,4 @@
Enable replay sanity checks on procedure tests.
</description>
</property>
<property>
<name>hbase.localcluster.assign.random.ports</name>
<value>true</value>
<description>
Assign random ports to master and RS info server (UI).
</description>
</property>
</configuration>