Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23993 Use loopback for zk standalone server in minizkcluster (#1291) 

hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
 Have client and server use loopback instead of 'localhost'

Signed-off-by: Duo Zhang <zhangduo@apache.org>
Signed-off-by: Jan Hentschel <janh@apache.org>
This commit is contained in:
Michael Stack 2020-03-17 20:14:24 -07:00 committed by GitHub
parent 292cadf847
commit ebd37a314c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 39 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-examples/src/test/java/org/apache/hadoop/hbase/coprocessor/example/TestZooKeeperScanPolicyObserver.java
View File

@ -66,7 +66,7 @@ public class TestZooKeeperScanPolicyObserver {
.createTable(TableDescriptorBuilder.newBuilder(NAME)
.setCoprocessor(ZooKeeperScanPolicyObserver.class.getName())
.setValue(ZooKeeperScanPolicyObserver.ZK_ENSEMBLE_KEY,
"localhost:" + UTIL.getZkCluster().getClientPort())
UTIL.getZkCluster().getAddress().toString())
.setValue(ZooKeeperScanPolicyObserver.ZK_SESSION_TIMEOUT_KEY, "2000")
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()).build());
TABLE = UTIL.getConnection().getTable(NAME);

7
hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.zookeeper.KeeperException;
import org.junit.Test;
@ -364,8 +365,8 @@ public abstract class TestReplicationStateBasic {
}
// Add peers for the corresponding queues so they are not orphans
rp.getPeerStorage().addPeer("qId" + i,
ReplicationPeerConfig.newBuilder().setClusterKey("localhost:2818:/bogus" + i).build(),
true);
ReplicationPeerConfig.newBuilder().
setClusterKey(MiniZooKeeperCluster.HOST + ":2818:/bogus" + i).build(), true);
}
}
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKConnectionRegistry.java
View File

@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@ -101,7 +102,7 @@ public class TestZKConnectionRegistry {
public void testIndependentZKConnections() throws IOException {
try (ReadOnlyZKClient zk1 = REGISTRY.getZKClient()) {
Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration());
otherConf.set(HConstants.ZOOKEEPER_QUORUM, "localhost");
otherConf.set(HConstants.ZOOKEEPER_QUORUM, MiniZooKeeperCluster.HOST);
try (ZKConnectionRegistry otherRegistry = new ZKConnectionRegistry(otherConf)) {
ReadOnlyZKClient zk2 = otherRegistry.getZKClient();
assertNotSame("Using a different configuration / quorum should result in different " +

3
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
View File

@ -25,7 +25,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.spy;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@ -78,7 +77,7 @@ public class TestReplicationProcedureRetry {
Admin admin = UTIL.getAdmin();
String peerId = "1";
ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
.setClusterKey("localhost:" + UTIL.getZkCluster().getClientPort() + ":/hbase2").build();
.setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/hbase2").build();
admin.addReplicationPeer(peerId, peerConfig, true);
assertEquals(peerConfig.getClusterKey(),

4
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
View File

@ -485,7 +485,7 @@ public abstract class TestReplicationSourceManager {
String replicationSourceImplName = conf.get("replication.replicationsource.implementation");
final String peerId = "FakePeer";
final ReplicationPeerConfig peerConfig = new ReplicationPeerConfig()
.setClusterKey("localhost:" + utility.getZkCluster().getClientPort() + ":/hbase");
.setClusterKey(utility.getZkCluster().getAddress().toString() + ":/hbase");
try {
DummyServer server = new DummyServer();
ReplicationQueueStorage rq = ReplicationStorageFactory
@ -540,7 +540,7 @@ public abstract class TestReplicationSourceManager {
public void testRemovePeerMetricsCleanup() throws Exception {
final String peerId = "DummyPeer";
final ReplicationPeerConfig peerConfig = new ReplicationPeerConfig()
.setClusterKey("localhost:" + utility.getZkCluster().getClientPort() + ":/hbase");
.setClusterKey(utility.getZkCluster().getAddress().toString() + ":/hbase");
try {
MetricsReplicationSourceSource globalSource = getGlobalSource();
final int globalLogQueueSizeInitial = globalSource.getSizeOfLogQueue();

9
hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java
View File

@ -31,7 +31,6 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
@ -62,7 +61,6 @@ import org.junit.runner.RunWith;
import org.mockito.ArgumentMatcher;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
@RunWith(MockitoJUnitRunner.class)
@ -309,8 +307,8 @@ public class TestCanaryTool {
private void testZookeeperCanaryWithArgs(String[] args) throws Exception {
Integer port =
Iterables.getOnlyElement(testingUtility.getZkCluster().getClientPortList(), null);
testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM,
"localhost:" + port + "/hbase");
String hostPort = testingUtility.getZkCluster().getAddress().toString();
testingUtility.getConfiguration().set(HConstants.ZOOKEEPER_QUORUM, hostPort + "/hbase");
ExecutorService executor = new ScheduledThreadPoolExecutor(2);
CanaryTool.ZookeeperStdOutSink sink = spy(new CanaryTool.ZookeeperStdOutSink());
CanaryTool canary = new CanaryTool(executor, sink);
@ -318,7 +316,6 @@ public class TestCanaryTool {
String baseZnode = testingUtility.getConfiguration()
.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
verify(sink, atLeastOnce())
.publishReadTiming(eq(baseZnode), eq("localhost:" + port), anyLong());
verify(sink, atLeastOnce()).publishReadTiming(eq(baseZnode), eq(hostPort), anyLong());
}
}

8
hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
View File

@ -575,7 +575,7 @@ module Hbase
end
define_test "set_peer_bandwidth: works with peer bandwidth upper limit" do
cluster_key = "localhost:2181:/hbase-test"
cluster_key = org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster::HOST + ":2181:/hbase-test"
args = { CLUSTER_KEY => cluster_key }
command(:add_peer, @peer_id, args)
@ -590,7 +590,7 @@ module Hbase
end
define_test "get_peer_config: works with simple clusterKey peer" do
cluster_key = "localhost:2181:/hbase-test"
cluster_key = org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster::HOST + ":2181:/hbase-test"
args = { CLUSTER_KEY => cluster_key }
command(:add_peer, @peer_id, args)
peer_config = command(:get_peer_config, @peer_id)
@ -600,7 +600,7 @@ module Hbase
end
define_test "get_peer_config: works with replicationendpointimpl peer and config params" do
cluster_key = 'localhost:2181:/hbase-test'
cluster_key = org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster::HOST + ":2181:/hbase-test"
repl_impl = 'org.apache.hadoop.hbase.replication.DummyReplicationEndpoint'
config_params = { "config1" => "value1", "config2" => "value2" }
args = { CLUSTER_KEY => cluster_key, ENDPOINT_CLASSNAME => repl_impl,
@ -616,7 +616,7 @@ module Hbase
end
define_test "list_peer_configs: returns all peers' ReplicationPeerConfig objects" do
cluster_key = "localhost:2181:/hbase-test"
cluster_key = org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster::HOST + ":2181:/hbase-test"
args = { CLUSTER_KEY => cluster_key }
peer_id_second = '2'
command(:add_peer, @peer_id, args)

22
hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.java
View File

@ -25,12 +25,14 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.BindException;
import java.net.ConnectException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.net.Address;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.server.NIOServerCnxnFactory;
@ -48,11 +50,12 @@ import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesti
@InterfaceAudience.Public
public class MiniZooKeeperCluster {
private static final Logger LOG = LoggerFactory.getLogger(MiniZooKeeperCluster.class);
private static final int TICK_TIME = 2000;
private static final int TIMEOUT = 1000;
private static final int DEFAULT_CONNECTION_TIMEOUT = 30000;
private int connectionTimeout;
public static final String LOOPBACK_HOST = InetAddress.getLoopbackAddress().getHostName();
public static final String HOST = LOOPBACK_HOST;
private boolean started;
@ -234,7 +237,7 @@ public class MiniZooKeeperCluster {
while (true) {
try {
standaloneServerFactory = new NIOServerCnxnFactory();
standaloneServerFactory.configure(new InetSocketAddress(currentClientPort),
standaloneServerFactory.configure(new InetSocketAddress(LOOPBACK_HOST, currentClientPort),
configuration.getInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS,
HConstants.DEFAULT_ZOOKEEPER_MAX_CLIENT_CNXNS));
} catch (BindException e) {
@ -416,7 +419,7 @@ public class MiniZooKeeperCluster {
long start = System.currentTimeMillis();
while (true) {
try {
send4LetterWord("localhost", port, "stat", (int)timeout);
send4LetterWord(HOST, port, "stat", (int)timeout);
} catch (IOException e) {
return true;
}
@ -439,7 +442,7 @@ public class MiniZooKeeperCluster {
long start = System.currentTimeMillis();
while (true) {
try {
String result = send4LetterWord("localhost", port, "stat", (int)timeout);
String result = send4LetterWord(HOST, port, "stat", (int)timeout);
if (result.startsWith("Zookeeper version:") && !result.contains("READ-ONLY")) {
return true;
} else {
@ -447,10 +450,10 @@ public class MiniZooKeeperCluster {
}
} catch (ConnectException e) {
// ignore as this is expected, do not log stacktrace
LOG.info("localhost:{} not up: {}", port, e.toString());
LOG.info("{}:{} not up: {}", HOST, port, e.toString());
} catch (IOException e) {
// ignore as this is expected
LOG.info("localhost:{} not up", port, e);
LOG.info("{}:{} not up", HOST, port, e);
}
if (System.currentTimeMillis() > start + timeout) {
@ -470,6 +473,13 @@ public class MiniZooKeeperCluster {
: clientPortList.get(activeZKServerIndex);
}
/**
* @return Address for this cluster instance.
*/
public Address getAddress() {
return Address.fromParts(HOST, getClientPort());
}
List<ZooKeeperServer> getZooKeeperServers() {
return zooKeeperServers;
}

8
hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestReadOnlyZKClient.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -36,7 +36,6 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
@ -83,8 +82,9 @@ public class TestReadOnlyZKClient {
@BeforeClass
public static void setUp() throws Exception {
final int port = UTIL.startMiniZKCluster().getClientPort();
String hostPort = UTIL.getZkCluster().getAddress().toString();
ZooKeeper zk = ZooKeeperHelper.getConnectedZooKeeper("localhost:" + port, 10000);
ZooKeeper zk = ZooKeeperHelper.getConnectedZooKeeper(hostPort, 10000);
DATA = new byte[10];
ThreadLocalRandom.current().nextBytes(DATA);
zk.create(PATH, DATA, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
@ -93,7 +93,7 @@ public class TestReadOnlyZKClient {
}
zk.close();
Configuration conf = UTIL.getConfiguration();
conf.set(HConstants.ZOOKEEPER_QUORUM, "localhost:" + port);
conf.set(HConstants.ZOOKEEPER_QUORUM, hostPort);
conf.setInt(ReadOnlyZKClient.RECOVERY_RETRY, 3);
conf.setInt(ReadOnlyZKClient.RECOVERY_RETRY_INTERVAL_MILLIS, 100);
conf.setInt(ReadOnlyZKClient.KEEPALIVE_MILLIS, 3000);

5
hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMainServer.java
View File

@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.zookeeper;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.security.Permission;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@ -83,8 +82,8 @@ public class TestZKMainServer {
ZKUtil.checkExists(zkw, znode);
boolean exception = false;
try {
ZKMainServer.main(new String [] {"-server",
"localhost:" + htu.getZkCluster().getClientPort(), "delete", znode});
ZKMainServer.main(new String [] {"-server", htu.getZkCluster().getAddress().toString(),
"delete", znode});
} catch (ExitException ee) {
// ZKMS calls System.exit which should trigger this exception.
exception = true;