HBASE-6036 Add Cluster-level PB-based calls to HMasterInterface (minus file-format related calls)

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1342108 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2012-05-24 00:33:48 +00:00
parent 44ddcab9a7
commit ca8882e804
10 changed files with 4031 additions and 57 deletions

View File

@ -76,7 +76,10 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.Bytes;
@ -1431,7 +1434,20 @@ public class HBaseAdmin implements Abortable, Closeable {
throws MasterNotRunningException, ZooKeeperConnectionException {
MasterKeepAliveConnection master = connection.getKeepAliveMaster();
try {
return master.balanceSwitch(b);
SetBalancerRunningRequest req = RequestConverter.buildLoadBalancerIsRequest(b, false);
return master.loadBalancerIs(null, req).getPrevBalanceValue();
} catch (ServiceException se) {
IOException ioe = ProtobufUtil.getRemoteException(se);
if (ioe instanceof MasterNotRunningException) {
throw (MasterNotRunningException)ioe;
}
if (ioe instanceof ZooKeeperConnectionException) {
throw (ZooKeeperConnectionException)ioe;
}
// Throwing MasterNotRunningException even though not really valid in order to not
// break interface by adding additional exception type.
throw new MasterNotRunningException("Unexpected exception when calling balanceSwitch",se);
} finally {
master.close();
}
@ -1444,10 +1460,10 @@ public class HBaseAdmin implements Abortable, Closeable {
* @return True if balancer ran, false otherwise.
*/
public boolean balancer()
throws MasterNotRunningException, ZooKeeperConnectionException {
throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException {
MasterKeepAliveConnection master = connection.getKeepAliveMaster();
try {
return master.balance();
return master.balance(null,RequestConverter.buildBalanceRequest()).getBalancerRan();
} finally {
master.close();
}
@ -1599,8 +1615,8 @@ public class HBaseAdmin implements Abortable, Closeable {
public synchronized void shutdown() throws IOException {
execute(new MasterCallable<Void>() {
@Override
public Void call() throws IOException {
master.shutdown();
public Void call() throws ServiceException {
master.shutdown(null,ShutdownRequest.newBuilder().build());
return null;
}
});
@ -1615,8 +1631,8 @@ public class HBaseAdmin implements Abortable, Closeable {
public synchronized void stopMaster() throws IOException {
execute(new MasterCallable<Void>() {
@Override
public Void call() throws IOException {
master.stopMaster();
public Void call() throws ServiceException {
master.stopMaster(null,StopMasterRequest.newBuilder().build());
return null;
}
});
@ -1678,7 +1694,7 @@ public class HBaseAdmin implements Abortable, Closeable {
* @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
public static void checkHBaseAvailable(Configuration conf)
throws MasterNotRunningException, ZooKeeperConnectionException {
throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException {
Configuration copyOfConf = HBaseConfiguration.create(conf);
// We set it to make it fail as soon as possible if HBase is not available
@ -1716,7 +1732,7 @@ public class HBaseAdmin implements Abortable, Closeable {
MasterKeepAliveConnection master = null;
try {
master = connection.getKeepAliveMaster();
master.isMasterRunning();
master.isMasterRunning(null,RequestConverter.buildIsMasterRunningRequest());
} finally {
if (master != null) {
master.close();

View File

@ -78,6 +78,8 @@ import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.VersionedProtocol;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.Bytes;
@ -672,7 +674,7 @@ public class HConnectionManager {
* Create a new Master proxy. Try once only.
*/
private HMasterInterface createMasterInterface()
throws IOException, KeeperException {
throws IOException, KeeperException, ServiceException {
ZooKeeperKeepAliveConnection zkw;
try {
@ -699,7 +701,8 @@ public class HConnectionManager {
HMasterInterface.class, HMasterInterface.VERSION, isa, this.conf,
this.rpcTimeout);
if (tryMaster.isMasterRunning()) {
if (tryMaster.isMasterRunning(
null, RequestConverter.buildIsMasterRunningRequest()).getIsMasterRunning()) {
return tryMaster;
} else {
HBaseRPC.stopProxy(tryMaster);
@ -760,6 +763,8 @@ public class HConnectionManager {
exceptionCaught = e;
} catch (KeeperException e) {
exceptionCaught = e;
} catch (ServiceException e) {
exceptionCaught = e;
}
if (exceptionCaught != null)
@ -1640,7 +1645,8 @@ public class HConnectionManager {
return false;
}
try {
return keepAliveMaster.isMasterRunning();
return keepAliveMaster.isMasterRunning(
null, RequestConverter.buildIsMasterRunningRequest()).getIsMasterRunning();
}catch (UndeclaredThrowableException e){
// It's somehow messy, but we can receive exceptions such as
// java.net.ConnectException but they're not declared. So we catch
@ -1648,6 +1654,9 @@ public class HConnectionManager {
LOG.info("Master connection is not running anymore",
e.getUndeclaredThrowable());
return false;
} catch (ServiceException se) {
LOG.warn("Checking master connection", se);
return false;
}
}

View File

@ -31,8 +31,19 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionReque
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterResponse;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.security.TokenInfo;
import org.apache.hadoop.hbase.security.KerberosInfo;
import org.apache.hadoop.hbase.util.Pair;
@ -67,8 +78,15 @@ public interface HMasterInterface extends VersionedProtocol {
// 31: 5/8/2012 - HBASE-5445: Converted to PB-based calls
public static final long VERSION = 31L;
/** @return true if master is available */
public boolean isMasterRunning();
/**
* @param c Unused (set to null).
* @param req IsMasterRunningRequest
* @return IsMasterRunningRequest that contains:<br>
* isMasterRunning: true if master is available
* @throws ServiceException
*/
public IsMasterRunningResponse isMasterRunning(RpcController c, IsMasterRunningRequest req)
throws ServiceException;
// Admin tools would use these cmds
@ -158,16 +176,24 @@ public interface HMasterInterface extends VersionedProtocol {
/**
* Shutdown an HBase cluster.
* @throws IOException e
* @param controller Unused (set to null).
* @param request ShutdownRequest
* @return ShutdownResponse
* @throws ServiceException
*/
public void shutdown() throws IOException;
public ShutdownResponse shutdown(RpcController controller, ShutdownRequest request)
throws ServiceException;
/**
* Stop HBase Master only.
* Does not shutdown the cluster.
* @throws IOException e
* @param controller Unused (set to null).
* @param request StopMasterRequest
* @return StopMasterResponse
* @throws ServiceException
*/
public void stopMaster() throws IOException;
public StopMasterResponse stopMaster(RpcController controller, StopMasterRequest request)
throws ServiceException;
/**
* Return cluster status.
@ -190,26 +216,27 @@ public interface HMasterInterface extends VersionedProtocol {
* Run the balancer. Will run the balancer and if regions to move, it will
* go ahead and do the reassignments. Can NOT run for various reasons. Check
* logs.
* @return True if balancer ran and was able to tell the region servers to
* @param c Unused (set to null).
* @param request BalanceRequest
* @return BalanceResponse that contains:<br>
* - balancerRan: True if balancer ran and was able to tell the region servers to
* unassign all the regions to balance (the re-assignment itself is async),
* false otherwise.
*/
public boolean balance();
public BalanceResponse balance(RpcController c, BalanceRequest request) throws ServiceException;
/**
* Turn the load balancer on or off.
* @param b If true, enable balancer. If false, disable balancer.
* @return Previous balancer value
* @param controller Unused (set to null).
* @param req SetBalancerRunningRequest that contains:<br>
* - on: If true, enable balancer. If false, disable balancer.<br>
* - synchronous: if true, wait until current balance() call, if outstanding, to return.
* @return SetBalancerRunningResponse that contains:<br>
* - prevBalanceValue: Previous balancer value
* @throws ServiceException
*/
public boolean balanceSwitch(final boolean b);
/**
* Turn the load balancer on or off.
* It waits until current balance() call, if outstanding, to return.
* @param b If true, enable balancer. If false, disable balancer.
* @return Previous balancer value
*/
public boolean synchronousBalanceSwitch(final boolean b);
public SetBalancerRunningResponse loadBalancerIs(RpcController controller, SetBalancerRunningRequest req)
throws ServiceException;
/**
* Get array of all HTDs.
@ -240,9 +267,9 @@ public interface HMasterInterface extends VersionedProtocol {
* back to the same server. Use {@link #moveRegion(RpcController,MoveRegionRequest}
* if you want to control the region movement.
* @param controller Unused (set to null).
* @param req The request which contains:
* @param req The request which contains:<br>
* - region: Region to unassign. Will clear any existing RegionPlan
* if one found.
* if one found.<br>
* - force: If true, force unassign (Will remove region from
* regions-in-transition too if present as well as from assigned regions --
* radical!.If results in double assignment use hbck -fix to resolve.
@ -254,11 +281,11 @@ public interface HMasterInterface extends VersionedProtocol {
/**
* Move a region to a specified destination server.
* @param controller Unused (set to null).
* @param req The request which contains:
* @param req The request which contains:<br>
* - region: The encoded region name; i.e. the hash that makes
* up the region name suffix: e.g. if regionname is
* <code>TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396.</code>,
* then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>.
* then the encoded region name is: <code>527db22f95c8a9e0116f0cc13c680396</code>.<br>
* - destServerName: The servername of the destination regionserver. If
* passed the empty byte array we'll assign to a random server. A server name
* is made of host, port and startcode. Here is an example:

View File

@ -79,6 +79,7 @@ import org.apache.hadoop.hbase.ipc.HBaseServer;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.RegionServerStatusProtocol;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.ipc.ProtocolSignature;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
@ -134,6 +135,16 @@ import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.Repor
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterResponse;
import com.google.protobuf.ServiceException;
/**
@ -1062,6 +1073,11 @@ Server {
return !isStopped();
}
public IsMasterRunningResponse isMasterRunning(RpcController c, IsMasterRunningRequest req)
throws ServiceException {
return IsMasterRunningResponse.newBuilder().setIsMasterRunning(isMasterRunning()).build();
}
/**
* @return Maximum time we should run balancer for
*/
@ -1079,7 +1095,6 @@ Server {
return balancerCutoffTime;
}
@Override
public boolean balance() {
// If balance not true, don't run balancer.
if (!this.balanceSwitch) return false;
@ -1154,6 +1169,11 @@ Server {
return balancerRan;
}
@Override
public BalanceResponse balance(RpcController c, BalanceRequest request) throws ServiceException {
return BalanceResponse.newBuilder().setBalancerRan(balance()).build();
}
enum BalanceSwitchMode {
SYNC,
ASYNC
@ -1185,19 +1205,25 @@ Server {
} catch (IOException ioe) {
LOG.warn("Error flipping balance switch", ioe);
}
return oldValue;
return oldValue;
}
@Override
public boolean synchronousBalanceSwitch(final boolean b) {
return switchBalancer(b, BalanceSwitchMode.SYNC);
}
@Override
public boolean balanceSwitch(final boolean b) {
return switchBalancer(b, BalanceSwitchMode.ASYNC);
}
@Override
public SetBalancerRunningResponse loadBalancerIs(RpcController controller, SetBalancerRunningRequest req)
throws ServiceException {
boolean prevValue = (req.getSynchronous())?
synchronousBalanceSwitch(req.getOn()):balanceSwitch(req.getOn());
return SetBalancerRunningResponse.newBuilder().setPrevBalanceValue(prevValue).build();
}
/**
* Switch for the background CatalogJanitor thread.
* Used for testing. The thread will continue to run. It will just be a noop
@ -1705,7 +1731,6 @@ Server {
}
@SuppressWarnings("deprecation")
@Override
public void shutdown() {
if (cpHost != null) {
try {
@ -1730,6 +1755,12 @@ Server {
}
@Override
public ShutdownResponse shutdown(RpcController controller, ShutdownRequest request)
throws ServiceException {
shutdown();
return ShutdownResponse.newBuilder().build();
}
public void stopMaster() {
if (cpHost != null) {
try {
@ -1741,6 +1772,13 @@ Server {
stop("Stopped by " + Thread.currentThread().getName());
}
@Override
public StopMasterResponse stopMaster(RpcController controller, StopMasterRequest request)
throws ServiceException {
stopMaster();
return StopMasterResponse.newBuilder().build();
}
@Override
public void stop(final String why) {
LOG.info(why);

View File

@ -81,6 +81,9 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Re
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
@ -884,4 +887,33 @@ public final class RequestConverter {
builder.setForce(force);
return builder.build();
}
/**
* Creates a protocol buffer IsMasterRunningRequest
*
* @return a IsMasterRunningRequest
*/
public static IsMasterRunningRequest buildIsMasterRunningRequest() {
return IsMasterRunningRequest.newBuilder().build();
}
/**
* Creates a protocol buffer BalanceRequest
*
* @return a BalanceRequest
*/
public static BalanceRequest buildBalanceRequest() {
return BalanceRequest.newBuilder().build();
}
/**
* Creates a protocol buffer SetBalancerRunningRequest
*
* @param on
* @param synchronous
* @return a SetBalancerRunningRequest
*/
public static SetBalancerRunningRequest buildLoadBalancerIsRequest(boolean on, boolean synchronous) {
return SetBalancerRunningRequest.newBuilder().setOn(on).setSynchronous(synchronous).build();
}
}

View File

@ -90,6 +90,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.TreeMultimap;
import com.google.protobuf.ServiceException;
/**
* HBaseFsck (hbck) is a tool for checking and repairing region consistency and
@ -376,7 +377,7 @@ public class HBaseFsck {
* Contacts the master and prints out cluster-wide information
* @return 0 on success, non-zero on failure
*/
public int onlineHbck() throws IOException, KeeperException, InterruptedException {
public int onlineHbck() throws IOException, KeeperException, InterruptedException, ServiceException {
// print hbase server version
errors.print("Version: " + status.getHBaseVersion());
offlineHdfsIntegrityRepair();

View File

@ -57,6 +57,8 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.junit.*;
import org.junit.experimental.categories.Category;
import com.google.protobuf.ServiceException;
/**
* Class to test HBaseAdmin.
@ -1574,6 +1576,7 @@ public class TestAdmin {
assertTrue(false);
} catch (MasterNotRunningException ignored) {
} catch (ZooKeeperConnectionException ignored) {
} catch (ServiceException ignored) {
}
long end = System.currentTimeMillis();

View File

@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.ipc.RemoteException;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@ -50,7 +51,7 @@ public class TestHMasterRPCException {
try {
HMasterInterface inf = (HMasterInterface) HBaseRPC.getProxy(
HMasterInterface.class, HMasterInterface.VERSION, isa, conf, 100);
inf.isMasterRunning();
inf.isMasterRunning(null,RequestConverter.buildIsMasterRunningRequest());
fail();
} catch (RemoteException ex) {
assertTrue(ex.getMessage().startsWith(

View File

@ -52,6 +52,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import com.google.protobuf.ServiceException;
/**
* Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction}
* only the below tests are against a running cluster where {@link TestSplitTransaction}
@ -104,7 +106,7 @@ public class TestSplitTransactionOnCluster {
* @throws DeserializationException
*/
@Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
throws IOException, InterruptedException, NodeExistsException, KeeperException, DeserializationException {
throws IOException, InterruptedException, NodeExistsException, KeeperException, DeserializationException, ServiceException {
final byte [] tableName =
Bytes.toBytes("ephemeral");
@ -174,7 +176,7 @@ public class TestSplitTransactionOnCluster {
}
@Test (timeout = 300000) public void testExistingZnodeBlocksSplitAndWeRollback()
throws IOException, InterruptedException, NodeExistsException, KeeperException {
throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException {
final byte [] tableName =
Bytes.toBytes("testExistingZnodeBlocksSplitAndWeRollback");
@ -234,7 +236,7 @@ public class TestSplitTransactionOnCluster {
* @throws InterruptedException
*/
@Test (timeout = 300000) public void testShutdownSimpleFixup()
throws IOException, InterruptedException {
throws IOException, InterruptedException, ServiceException {
final byte [] tableName = Bytes.toBytes("testShutdownSimpleFixup");
// Create table then get the single region for our new table.
@ -290,7 +292,7 @@ public class TestSplitTransactionOnCluster {
* @throws InterruptedException
*/
@Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
throws IOException, InterruptedException {
throws IOException, InterruptedException, ServiceException {
final byte [] tableName =
Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
@ -371,7 +373,7 @@ public class TestSplitTransactionOnCluster {
@Test(timeout = 300000)
public void testMasterRestartWhenSplittingIsPartial()
throws IOException, InterruptedException, NodeExistsException,
KeeperException, DeserializationException {
KeeperException, DeserializationException, ServiceException {
final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
// Create table then get the single region for our new table.
@ -451,7 +453,7 @@ public class TestSplitTransactionOnCluster {
@Test (timeout = 300000)
public void testMasterRestartAtRegionSplitPendingCatalogJanitor()
throws IOException, InterruptedException, NodeExistsException,
KeeperException {
KeeperException, ServiceException {
final byte[] tableName = Bytes.toBytes("testMasterRestartAtRegionSplitPendingCatalogJanitor");
// Create table then get the single region for our new table.