diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java index 17552402127..47a2b7d1e24 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.MetaRegionServer; import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.Bytes; @@ -54,6 +55,8 @@ import java.util.regex.Pattern; @InterfaceAudience.Public @InterfaceStability.Evolving public class ServerName implements Comparable, Serializable { + private static final long serialVersionUID = 1367463982557264981L; + /** * Version for this class. * Its a short rather than a byte so I can for sure distinguish between this @@ -370,9 +373,9 @@ public class ServerName implements Comparable, Serializable { if (ProtobufUtil.isPBMagicPrefix(data)) { int prefixLen = ProtobufUtil.lengthOfPBMagic(); try { - MetaRegionServer rss = - MetaRegionServer.PARSER.parseFrom(data, prefixLen, data.length - prefixLen); - org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn = rss.getServer(); + ZooKeeperProtos.Master rss = + ZooKeeperProtos.Master.PARSER.parseFrom(data, prefixLen, data.length - prefixLen); + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn = rss.getMaster(); return valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode()); } catch (InvalidProtocolBufferException e) { // A failed parse of the znode is pretty catastrophic. Rather than loop diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java index 6ee796a25a1..ec9619635c6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java @@ -53,9 +53,125 @@ public class RegionState { SPLITTING_NEW, // new region to be created when RS splits a parent // region but hasn't be created yet, or master doesn't // know it's already created - MERGING_NEW // new region to be created when RS merges two + MERGING_NEW; // new region to be created when RS merges two // daughter regions but hasn't be created yet, or // master doesn't know it's already created + + /** + * Convert to protobuf ClusterStatusProtos.RegionState.State + */ + public ClusterStatusProtos.RegionState.State convert() { + ClusterStatusProtos.RegionState.State rs; + switch (this) { + case OFFLINE: + rs = ClusterStatusProtos.RegionState.State.OFFLINE; + break; + case PENDING_OPEN: + rs = ClusterStatusProtos.RegionState.State.PENDING_OPEN; + break; + case OPENING: + rs = ClusterStatusProtos.RegionState.State.OPENING; + break; + case OPEN: + rs = ClusterStatusProtos.RegionState.State.OPEN; + break; + case PENDING_CLOSE: + rs = ClusterStatusProtos.RegionState.State.PENDING_CLOSE; + break; + case CLOSING: + rs = ClusterStatusProtos.RegionState.State.CLOSING; + break; + case CLOSED: + rs = ClusterStatusProtos.RegionState.State.CLOSED; + break; + case SPLITTING: + rs = ClusterStatusProtos.RegionState.State.SPLITTING; + break; + case SPLIT: + rs = ClusterStatusProtos.RegionState.State.SPLIT; + break; + case FAILED_OPEN: + rs = ClusterStatusProtos.RegionState.State.FAILED_OPEN; + break; + case FAILED_CLOSE: + rs = ClusterStatusProtos.RegionState.State.FAILED_CLOSE; + break; + case MERGING: + rs = ClusterStatusProtos.RegionState.State.MERGING; + break; + case MERGED: + rs = ClusterStatusProtos.RegionState.State.MERGED; + break; + case SPLITTING_NEW: + rs = ClusterStatusProtos.RegionState.State.SPLITTING_NEW; + break; + case MERGING_NEW: + rs = ClusterStatusProtos.RegionState.State.MERGING_NEW; + break; + default: + throw new IllegalStateException(""); + } + return rs; + } + + /** + * Convert a protobuf HBaseProtos.RegionState.State to a RegionState.State + * + * @return the RegionState.State + */ + public static State convert(ClusterStatusProtos.RegionState.State protoState) { + State state; + switch (protoState) { + case OFFLINE: + state = OFFLINE; + break; + case PENDING_OPEN: + state = PENDING_OPEN; + break; + case OPENING: + state = OPENING; + break; + case OPEN: + state = OPEN; + break; + case PENDING_CLOSE: + state = PENDING_CLOSE; + break; + case CLOSING: + state = CLOSING; + break; + case CLOSED: + state = CLOSED; + break; + case SPLITTING: + state = SPLITTING; + break; + case SPLIT: + state = SPLIT; + break; + case FAILED_OPEN: + state = FAILED_OPEN; + break; + case FAILED_CLOSE: + state = FAILED_CLOSE; + break; + case MERGING: + state = MERGING; + break; + case MERGED: + state = MERGED; + break; + case SPLITTING_NEW: + state = SPLITTING_NEW; + break; + case MERGING_NEW: + state = MERGING_NEW; + break; + default: + throw new IllegalStateException(""); + } + return state; + } } // Many threads can update the state at the stamp at the same time @@ -264,58 +380,8 @@ public class RegionState { */ public ClusterStatusProtos.RegionState convert() { ClusterStatusProtos.RegionState.Builder regionState = ClusterStatusProtos.RegionState.newBuilder(); - ClusterStatusProtos.RegionState.State rs; - switch (this.state) { - case OFFLINE: - rs = ClusterStatusProtos.RegionState.State.OFFLINE; - break; - case PENDING_OPEN: - rs = ClusterStatusProtos.RegionState.State.PENDING_OPEN; - break; - case OPENING: - rs = ClusterStatusProtos.RegionState.State.OPENING; - break; - case OPEN: - rs = ClusterStatusProtos.RegionState.State.OPEN; - break; - case PENDING_CLOSE: - rs = ClusterStatusProtos.RegionState.State.PENDING_CLOSE; - break; - case CLOSING: - rs = ClusterStatusProtos.RegionState.State.CLOSING; - break; - case CLOSED: - rs = ClusterStatusProtos.RegionState.State.CLOSED; - break; - case SPLITTING: - rs = ClusterStatusProtos.RegionState.State.SPLITTING; - break; - case SPLIT: - rs = ClusterStatusProtos.RegionState.State.SPLIT; - break; - case FAILED_OPEN: - rs = ClusterStatusProtos.RegionState.State.FAILED_OPEN; - break; - case FAILED_CLOSE: - rs = ClusterStatusProtos.RegionState.State.FAILED_CLOSE; - break; - case MERGING: - rs = ClusterStatusProtos.RegionState.State.MERGING; - break; - case MERGED: - rs = ClusterStatusProtos.RegionState.State.MERGED; - break; - case SPLITTING_NEW: - rs = ClusterStatusProtos.RegionState.State.SPLITTING_NEW; - break; - case MERGING_NEW: - rs = ClusterStatusProtos.RegionState.State.MERGING_NEW; - break; - default: - throw new IllegalStateException(""); - } regionState.setRegionInfo(HRegionInfo.convert(hri)); - regionState.setState(rs); + regionState.setState(state.convert()); regionState.setStamp(getStamp()); return regionState.build(); } @@ -326,58 +392,8 @@ public class RegionState { * @return the RegionState */ public static RegionState convert(ClusterStatusProtos.RegionState proto) { - RegionState.State state; - switch (proto.getState()) { - case OFFLINE: - state = State.OFFLINE; - break; - case PENDING_OPEN: - state = State.PENDING_OPEN; - break; - case OPENING: - state = State.OPENING; - break; - case OPEN: - state = State.OPEN; - break; - case PENDING_CLOSE: - state = State.PENDING_CLOSE; - break; - case CLOSING: - state = State.CLOSING; - break; - case CLOSED: - state = State.CLOSED; - break; - case SPLITTING: - state = State.SPLITTING; - break; - case SPLIT: - state = State.SPLIT; - break; - case FAILED_OPEN: - state = State.FAILED_OPEN; - break; - case FAILED_CLOSE: - state = State.FAILED_CLOSE; - break; - case MERGING: - state = State.MERGING; - break; - case MERGED: - state = State.MERGED; - break; - case SPLITTING_NEW: - state = State.SPLITTING_NEW; - break; - case MERGING_NEW: - state = State.MERGING_NEW; - break; - default: - throw new IllegalStateException(""); - } - - return new RegionState(HRegionInfo.convert(proto.getRegionInfo()),state,proto.getStamp(),null); + return new RegionState(HRegionInfo.convert(proto.getRegionInfo()), + State.convert(proto.getState()), proto.getStamp(), null); } protected void setTimestamp(final long timestamp) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java index 68aff851707..752053cbcbe 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.zookeeper; import com.google.common.base.Stopwatch; +import com.google.protobuf.InvalidProtocolBufferException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -30,11 +32,13 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.MetaRegionServer; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -52,6 +56,8 @@ import java.rmi.UnknownHostException; import java.util.List; import java.util.ArrayList; +import javax.annotation.Nullable; + /** * Utility class to perform operation (get/wait for/verify/set/delete) on znode in ZooKeeper * which keeps hbase:meta region server location. @@ -121,18 +127,13 @@ public class MetaTableLocator { * @param zkw zookeeper connection to use * @return server name or null if we failed to get the data. */ + @Nullable public ServerName getMetaRegionLocation(final ZooKeeperWatcher zkw) { try { - try { - return ServerName.parseFrom(ZKUtil.getData(zkw, zkw.metaServerZNode)); - } catch (DeserializationException e) { - throw ZKUtil.convert(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return null; - } + RegionState state = getMetaRegionState(zkw); + return state.isOpened() ? state.getServerName() : null; } catch (KeeperException ke) { - return null; + return null; } } @@ -216,9 +217,8 @@ public class MetaTableLocator { } catch (RegionServerStoppedException e) { // Pass -- server name sends us to a server that is dying or already dead. } - return (service == null)? false: - verifyRegionLocation(service, - getMetaRegionLocation(zkw), META_REGION_NAME); + return (service != null) && verifyRegionLocation(service, + getMetaRegionLocation(zkw), META_REGION_NAME); } /** @@ -342,44 +342,65 @@ public class MetaTableLocator { * Sets the location of hbase:meta in ZooKeeper to the * specified server address. * @param zookeeper zookeeper reference - * @param location The server hosting hbase:meta + * @param serverName The server hosting hbase:meta + * @param state The region transition state * @throws KeeperException unexpected zookeeper exception */ public static void setMetaLocation(ZooKeeperWatcher zookeeper, - final ServerName location) - throws KeeperException { - LOG.info("Setting hbase:meta region location in ZooKeeper as " + location); + ServerName serverName, RegionState.State state) throws KeeperException { + LOG.info("Setting hbase:meta region location in ZooKeeper as " + serverName); // Make the MetaRegionServer pb and then get its bytes and save this as // the znode content. - byte [] data = toByteArray(location); + MetaRegionServer pbrsr = MetaRegionServer.newBuilder() + .setServer(ProtobufUtil.toServerName(serverName)) + .setRpcVersion(HConstants.RPC_CURRENT_VERSION) + .setState(state.convert()).build(); + byte[] data = ProtobufUtil.prependPBMagic(pbrsr.toByteArray()); try { - ZKUtil.createAndWatch(zookeeper, zookeeper.metaServerZNode, data); - } catch(KeeperException.NodeExistsException nee) { - LOG.debug("META region location already existed, updated location"); ZKUtil.setData(zookeeper, zookeeper.metaServerZNode, data); + } catch(KeeperException.NoNodeException nne) { + LOG.debug("META region location doesn't existed, create it"); + ZKUtil.createAndWatch(zookeeper, zookeeper.metaServerZNode, data); } } /** - * Build up the znode content. - * @param sn What to put into the znode. - * @return The content of the meta-region-server znode + * Load the meta region state from the meta server ZNode. */ - private static byte [] toByteArray(final ServerName sn) { - // ZNode content is a pb message preceded by some pb magic. - HBaseProtos.ServerName pbsn = - HBaseProtos.ServerName.newBuilder() - .setHostName(sn.getHostname()) - .setPort(sn.getPort()) - .setStartCode(sn.getStartcode()) - .build(); - - ZooKeeperProtos.MetaRegionServer pbrsr = - ZooKeeperProtos.MetaRegionServer.newBuilder() - .setServer(pbsn) - .setRpcVersion(HConstants.RPC_CURRENT_VERSION) - .build(); - return ProtobufUtil.prependPBMagic(pbrsr.toByteArray()); + public static RegionState getMetaRegionState(ZooKeeperWatcher zkw) throws KeeperException { + RegionState.State state = RegionState.State.OPEN; + ServerName serverName = null; + try { + byte[] data = ZKUtil.getData(zkw, zkw.metaServerZNode); + if (data != null && data.length > 0 && ProtobufUtil.isPBMagicPrefix(data)) { + try { + int prefixLen = ProtobufUtil.lengthOfPBMagic(); + ZooKeeperProtos.MetaRegionServer rl = + ZooKeeperProtos.MetaRegionServer.PARSER.parseFrom + (data, prefixLen, data.length - prefixLen); + if (rl.hasState()) { + state = RegionState.State.convert(rl.getState()); + } + HBaseProtos.ServerName sn = rl.getServer(); + serverName = ServerName.valueOf( + sn.getHostName(), sn.getPort(), sn.getStartCode()); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException("Unable to parse meta region location"); + } + } else { + // old style of meta region location? + serverName = ServerName.parseFrom(data); + } + } catch (DeserializationException e) { + throw ZKUtil.convert(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (serverName == null) { + state = RegionState.State.OFFLINE; + } + return new RegionState(HRegionInfo.FIRST_META_REGIONINFO, + state, serverName); } /** @@ -389,7 +410,7 @@ public class MetaTableLocator { */ public void deleteMetaLocation(ZooKeeperWatcher zookeeper) throws KeeperException { - LOG.info("Unsetting hbase:meta region location in ZooKeeper"); + LOG.info("Deleting hbase:meta region location in ZooKeeper"); try { // Just delete the node. Don't need any watches. ZKUtil.deleteNode(zookeeper, zookeeper.metaServerZNode); @@ -399,7 +420,7 @@ public class MetaTableLocator { } /** - * Wait until the meta region is available. + * Wait until the meta region is available and is not in transition. * @param zkw zookeeper connection to use * @param timeout maximum time to wait, in millis * @return ServerName or null if we timed out. @@ -408,14 +429,23 @@ public class MetaTableLocator { public ServerName blockUntilAvailable(final ZooKeeperWatcher zkw, final long timeout) throws InterruptedException { - byte [] data = ZKUtil.blockUntilAvailable(zkw, zkw.metaServerZNode, timeout); - if (data == null) return null; + if (timeout < 0) throw new IllegalArgumentException(); + if (zkw == null) throw new IllegalArgumentException(); + Stopwatch sw = new Stopwatch().start(); + ServerName sn = null; try { - return ServerName.parseFrom(data); - } catch (DeserializationException e) { - LOG.warn("Failed parse", e); - return null; + while (true) { + sn = getMetaRegionLocation(zkw); + if (sn != null || sw.elapsedMillis() + > timeout - HConstants.SOCKET_RETRY_WAIT_MS) { + break; + } + Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS); + } + } finally { + sw.stop(); } + return sn; } /** diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java index fb035392fef..8bad13f636f 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java @@ -16,7 +16,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      *
*/ boolean hasServer(); @@ -24,7 +25,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      *
*/ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServer(); @@ -32,7 +34,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      *
*/ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerOrBuilder(); @@ -58,6 +61,24 @@ public final class ZooKeeperProtos { * */ int getRpcVersion(); + + // optional .RegionState.State state = 3; + /** + * optional .RegionState.State state = 3; + * + * 
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     *
+ */ + boolean hasState(); + /** + * optional .RegionState.State state = 3; + * + * 
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     *
+ */ + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState(); } /** * Protobuf type {@code MetaRegionServer} @@ -133,6 +154,17 @@ public final class ZooKeeperProtos { rpcVersion_ = input.readUInt32(); break; } + case 24: { + int rawValue = input.readEnum(); + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State value = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(3, rawValue); + } else { + bitField0_ |= 0x00000004; + state_ = value; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -180,7 +212,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      *
*/ public boolean hasServer() { @@ -190,7 +223,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      *
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServer() { @@ -200,7 +234,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      *
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerOrBuilder() { @@ -235,9 +270,34 @@ public final class ZooKeeperProtos { return rpcVersion_; } + // optional .RegionState.State state = 3; + public static final int STATE_FIELD_NUMBER = 3; + private org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State state_; + /** + * optional .RegionState.State state = 3; + * + * 
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     *
+ */ + public boolean hasState() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional .RegionState.State state = 3; + * + * 
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     *
+ */ + public org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState() { + return state_; + } + private void initFields() { server_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance(); rpcVersion_ = 0; + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -265,6 +325,9 @@ public final class ZooKeeperProtos { if (((bitField0_ & 0x00000002) == 0x00000002)) { output.writeUInt32(2, rpcVersion_); } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeEnum(3, state_.getNumber()); + } getUnknownFields().writeTo(output); } @@ -282,6 +345,10 @@ public final class ZooKeeperProtos { size += com.google.protobuf.CodedOutputStream .computeUInt32Size(2, rpcVersion_); } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(3, state_.getNumber()); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -315,6 +382,11 @@ public final class ZooKeeperProtos { result = result && (getRpcVersion() == other.getRpcVersion()); } + result = result && (hasState() == other.hasState()); + if (hasState()) { + result = result && + (getState() == other.getState()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -336,6 +408,10 @@ public final class ZooKeeperProtos { hash = (37 * hash) + RPC_VERSION_FIELD_NUMBER; hash = (53 * hash) + getRpcVersion(); } + if (hasState()) { + hash = (37 * hash) + STATE_FIELD_NUMBER; + hash = (53 * hash) + hashEnum(getState()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -459,6 +535,8 @@ public final class ZooKeeperProtos { bitField0_ = (bitField0_ & ~0x00000001); rpcVersion_ = 0; bitField0_ = (bitField0_ & ~0x00000002); + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + bitField0_ = (bitField0_ & ~0x00000004); return this; } @@ -499,6 +577,10 @@ public final class ZooKeeperProtos { to_bitField0_ |= 0x00000002; } result.rpcVersion_ = rpcVersion_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.state_ = state_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -521,6 +603,9 @@ public final class ZooKeeperProtos { if (other.hasRpcVersion()) { setRpcVersion(other.getRpcVersion()); } + if (other.hasState()) { + setState(other.getState()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -564,7 +649,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ public boolean hasServer() { @@ -574,7 +660,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServer() { @@ -588,7 +675,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ public Builder setServer(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { @@ -608,7 +696,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ public Builder setServer( @@ -626,7 +715,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ public Builder mergeServer(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { @@ -649,7 +739,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ public Builder clearServer() { @@ -666,7 +757,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder getServerBuilder() { @@ -678,7 +770,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerOrBuilder() { @@ -692,7 +785,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * * 
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        *
*/ private com.google.protobuf.SingleFieldBuilder< @@ -766,6 +860,58 @@ public final class ZooKeeperProtos { return this; } + // optional .RegionState.State state = 3; + private org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + /** + * optional .RegionState.State state = 3; + * + * 
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       *
+ */ + public boolean hasState() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional .RegionState.State state = 3; + * + * 
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       *
+ */ + public org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState() { + return state_; + } + /** + * optional .RegionState.State state = 3; + * + * 
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       *
+ */ + public Builder setState(org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + state_ = value; + onChanged(); + return this; + } + /** + * optional .RegionState.State state = 3; + * + * 
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       *
+ */ + public Builder clearState() { + bitField0_ = (bitField0_ & ~0x00000004); + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:MetaRegionServer) } @@ -10596,43 +10742,44 @@ public final class ZooKeeperProtos { descriptor; static { java.lang.String[] descriptorData = { - "\n\017ZooKeeper.proto\032\013HBase.proto\"D\n\020MetaRe" + - "gionServer\022\033\n\006server\030\001 \002(\0132\013.ServerName\022" + - "\023\n\013rpc_version\030\002 \001(\r\"M\n\006Master\022\033\n\006master" + - "\030\001 \002(\0132\013.ServerName\022\023\n\013rpc_version\030\002 \001(\r" + - "\022\021\n\tinfo_port\030\003 \001(\r\"\037\n\tClusterUp\022\022\n\nstar" + - "t_date\030\001 \002(\t\"\210\001\n\020RegionTransition\022\027\n\017eve" + - "nt_type_code\030\001 \002(\r\022\023\n\013region_name\030\002 \002(\014\022" + - "\023\n\013create_time\030\003 \002(\004\022 \n\013server_name\030\004 \002(" + - "\0132\013.ServerName\022\017\n\007payload\030\005 \001(\014\"\214\002\n\014Spli" + - "tLogTask\022\"\n\005state\030\001 \002(\0162\023.SplitLogTask.S", - "tate\022 \n\013server_name\030\002 \002(\0132\013.ServerName\0221" + - "\n\004mode\030\003 \001(\0162\032.SplitLogTask.RecoveryMode" + - ":\007UNKNOWN\"C\n\005State\022\016\n\nUNASSIGNED\020\000\022\t\n\005OW" + - "NED\020\001\022\014\n\010RESIGNED\020\002\022\010\n\004DONE\020\003\022\007\n\003ERR\020\004\">" + - "\n\014RecoveryMode\022\013\n\007UNKNOWN\020\000\022\021\n\rLOG_SPLIT" + - "TING\020\001\022\016\n\nLOG_REPLAY\020\002\"n\n\005Table\022$\n\005state" + - "\030\001 \002(\0162\014.Table.State:\007ENABLED\"?\n\005State\022\013" + - "\n\007ENABLED\020\000\022\014\n\010DISABLED\020\001\022\r\n\tDISABLING\020\002" + - "\022\014\n\010ENABLING\020\003\"\215\001\n\017ReplicationPeer\022\022\n\ncl" + - "usterkey\030\001 \002(\t\022\037\n\027replicationEndpointImp", - "l\030\002 \001(\t\022\035\n\004data\030\003 \003(\0132\017.BytesBytesPair\022&" + - "\n\rconfiguration\030\004 \003(\0132\017.NameStringPair\"^" + - "\n\020ReplicationState\022&\n\005state\030\001 \002(\0162\027.Repl" + - "icationState.State\"\"\n\005State\022\013\n\007ENABLED\020\000" + - "\022\014\n\010DISABLED\020\001\"+\n\027ReplicationHLogPositio" + - "n\022\020\n\010position\030\001 \002(\003\"%\n\017ReplicationLock\022\022" + - "\n\nlock_owner\030\001 \002(\t\"\230\001\n\tTableLock\022\036\n\ntabl" + - "e_name\030\001 \001(\0132\n.TableName\022\037\n\nlock_owner\030\002" + - " \001(\0132\013.ServerName\022\021\n\tthread_id\030\003 \001(\003\022\021\n\t" + - "is_shared\030\004 \001(\010\022\017\n\007purpose\030\005 \001(\t\022\023\n\013crea", - "te_time\030\006 \001(\003\";\n\017StoreSequenceId\022\023\n\013fami" + - "ly_name\030\001 \002(\014\022\023\n\013sequence_id\030\002 \002(\004\"g\n\026Re" + - "gionStoreSequenceIds\022 \n\030last_flushed_seq" + - "uence_id\030\001 \002(\004\022+\n\021store_sequence_id\030\002 \003(" + - "\0132\020.StoreSequenceIdBE\n*org.apache.hadoop" + - ".hbase.protobuf.generatedB\017ZooKeeperProt" + - "osH\001\210\001\001\240\001\001" + "\n\017ZooKeeper.proto\032\013HBase.proto\032\023ClusterS" + + "tatus.proto\"g\n\020MetaRegionServer\022\033\n\006serve" + + "r\030\001 \002(\0132\013.ServerName\022\023\n\013rpc_version\030\002 \001(" + + "\r\022!\n\005state\030\003 \001(\0162\022.RegionState.State\"M\n\006" + + "Master\022\033\n\006master\030\001 \002(\0132\013.ServerName\022\023\n\013r" + + "pc_version\030\002 \001(\r\022\021\n\tinfo_port\030\003 \001(\r\"\037\n\tC" + + "lusterUp\022\022\n\nstart_date\030\001 \002(\t\"\210\001\n\020RegionT" + + "ransition\022\027\n\017event_type_code\030\001 \002(\r\022\023\n\013re" + + "gion_name\030\002 \002(\014\022\023\n\013create_time\030\003 \002(\004\022 \n\013" + + "server_name\030\004 \002(\0132\013.ServerName\022\017\n\007payloa", + "d\030\005 \001(\014\"\214\002\n\014SplitLogTask\022\"\n\005state\030\001 \002(\0162" + + "\023.SplitLogTask.State\022 \n\013server_name\030\002 \002(" + + "\0132\013.ServerName\0221\n\004mode\030\003 \001(\0162\032.SplitLogT" + + "ask.RecoveryMode:\007UNKNOWN\"C\n\005State\022\016\n\nUN" + + "ASSIGNED\020\000\022\t\n\005OWNED\020\001\022\014\n\010RESIGNED\020\002\022\010\n\004D" + + "ONE\020\003\022\007\n\003ERR\020\004\">\n\014RecoveryMode\022\013\n\007UNKNOW" + + "N\020\000\022\021\n\rLOG_SPLITTING\020\001\022\016\n\nLOG_REPLAY\020\002\"n" + + "\n\005Table\022$\n\005state\030\001 \002(\0162\014.Table.State:\007EN" + + "ABLED\"?\n\005State\022\013\n\007ENABLED\020\000\022\014\n\010DISABLED\020" + + "\001\022\r\n\tDISABLING\020\002\022\014\n\010ENABLING\020\003\"\215\001\n\017Repli", + "cationPeer\022\022\n\nclusterkey\030\001 \002(\t\022\037\n\027replic" + + "ationEndpointImpl\030\002 \001(\t\022\035\n\004data\030\003 \003(\0132\017." + + "BytesBytesPair\022&\n\rconfiguration\030\004 \003(\0132\017." + + "NameStringPair\"^\n\020ReplicationState\022&\n\005st" + + "ate\030\001 \002(\0162\027.ReplicationState.State\"\"\n\005St" + + "ate\022\013\n\007ENABLED\020\000\022\014\n\010DISABLED\020\001\"+\n\027Replic" + + "ationHLogPosition\022\020\n\010position\030\001 \002(\003\"%\n\017R" + + "eplicationLock\022\022\n\nlock_owner\030\001 \002(\t\"\230\001\n\tT" + + "ableLock\022\036\n\ntable_name\030\001 \001(\0132\n.TableName" + + "\022\037\n\nlock_owner\030\002 \001(\0132\013.ServerName\022\021\n\tthr", + "ead_id\030\003 \001(\003\022\021\n\tis_shared\030\004 \001(\010\022\017\n\007purpo" + + "se\030\005 \001(\t\022\023\n\013create_time\030\006 \001(\003\";\n\017StoreSe" + + "quenceId\022\023\n\013family_name\030\001 \002(\014\022\023\n\013sequenc" + + "e_id\030\002 \002(\004\"g\n\026RegionStoreSequenceIds\022 \n\030" + + "last_flushed_sequence_id\030\001 \002(\004\022+\n\021store_" + + "sequence_id\030\002 \003(\0132\020.StoreSequenceIdBE\n*o" + + "rg.apache.hadoop.hbase.protobuf.generate" + + "dB\017ZooKeeperProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -10644,7 +10791,7 @@ public final class ZooKeeperProtos { internal_static_MetaRegionServer_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_MetaRegionServer_descriptor, - new java.lang.String[] { "Server", "RpcVersion", }); + new java.lang.String[] { "Server", "RpcVersion", "State", }); internal_static_Master_descriptor = getDescriptor().getMessageTypes().get(1); internal_static_Master_fieldAccessorTable = new @@ -10724,6 +10871,7 @@ public final class ZooKeeperProtos { .internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[] { org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.getDescriptor(), + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.getDescriptor(), }, assigner); } diff --git a/hbase-protocol/src/main/protobuf/ZooKeeper.proto b/hbase-protocol/src/main/protobuf/ZooKeeper.proto index 03fbcd741fa..2ccfe8d7116 100644 --- a/hbase-protocol/src/main/protobuf/ZooKeeper.proto +++ b/hbase-protocol/src/main/protobuf/ZooKeeper.proto @@ -26,17 +26,22 @@ option java_generate_equals_and_hash = true; option optimize_for = SPEED; import "HBase.proto"; +import "ClusterStatus.proto"; /** * Content of the meta-region-server znode. */ message MetaRegionServer { - // The ServerName hosting the meta region currently. + // The ServerName hosting the meta region currently, or destination server, + // if meta region is in transition. required ServerName server = 1; // The major version of the rpc the server speaks. This is used so that // clients connecting to the cluster can have prior knowledge of what version // to send to a RegionServer. AsyncHBase will use this to detect versions. optional uint32 rpc_version = 2; + + // State of the region transition. OPEN means fully operational 'hbase:meta' + optional RegionState.State state = 3; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 034c4f615e9..fc3571c4617 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -764,7 +764,7 @@ public class AssignmentManager extends ZooKeeperListener { if (regionInfo.isMetaRegion()) { // If it's meta region, reset the meta location. // So that master knows the right meta region server. - MetaTableLocator.setMetaLocation(watcher, sn); + MetaTableLocator.setMetaLocation(watcher, sn, State.OPEN); } else { // No matter the previous server is online or offline, // we need to reset the last region server of the region. @@ -2947,31 +2947,35 @@ public class AssignmentManager extends ZooKeeperListener { processRegionInTransition(encodedRegionName, null); } } else if (!useZKForAssignment) { - // We need to send RPC call again for PENDING_OPEN/PENDING_CLOSE regions - // in case the RPC call is not sent out yet before the master was shut down - // since we update the state before we send the RPC call. We can't update - // the state after the RPC call. Otherwise, we don't know what's happened - // to the region if the master dies right after the RPC call is out. - Map rits = regionStates.getRegionsInTransition(); - for (RegionState regionState: rits.values()) { - if (!serverManager.isServerOnline(regionState.getServerName())) { - continue; // SSH will handle it - } - State state = regionState.getState(); - LOG.info("Processing " + regionState); - switch (state) { - case CLOSED: - invokeAssign(regionState.getRegion()); - break; - case PENDING_OPEN: - retrySendRegionOpen(regionState); - break; - case PENDING_CLOSE: - retrySendRegionClose(regionState); - break; - default: - // No process for other states - } + processRegionInTransitionZkLess(); + } + } + + void processRegionInTransitionZkLess() { + // We need to send RPC call again for PENDING_OPEN/PENDING_CLOSE regions + // in case the RPC call is not sent out yet before the master was shut down + // since we update the state before we send the RPC call. We can't update + // the state after the RPC call. Otherwise, we don't know what's happened + // to the region if the master dies right after the RPC call is out. + Map rits = regionStates.getRegionsInTransition(); + for (RegionState regionState: rits.values()) { + if (!serverManager.isServerOnline(regionState.getServerName())) { + continue; // SSH will handle it + } + State state = regionState.getState(); + LOG.info("Processing " + regionState); + switch (state) { + case CLOSED: + invokeAssign(regionState.getRegion()); + break; + case PENDING_OPEN: + retrySendRegionOpen(regionState); + break; + case PENDING_CLOSE: + retrySendRegionClose(regionState); + break; + default: + // No process for other states } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index f7b34da58d0..7884a417fc9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -112,6 +112,7 @@ import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CompressionTest; +import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.Pair; @@ -703,17 +704,26 @@ public class HMaster extends HRegionServer implements MasterServices, Server { status.setStatus("Assigning hbase:meta region"); RegionStates regionStates = assignmentManager.getRegionStates(); - regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO); + RegionState regionState = MetaTableLocator.getMetaRegionState(this.getZooKeeper()); + ServerName currentMetaServer = regionState.getServerName(); + if (!ConfigUtil.useZKForAssignment(conf)) { + regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO, regionState.getState(), + currentMetaServer, null); + } else { + regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO); + } boolean rit = this.assignmentManager - .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO); + .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO); boolean metaRegionLocation = metaTableLocator.verifyMetaRegionLocation( this.getShortCircuitConnection(), this.getZooKeeper(), timeout); - ServerName currentMetaServer = metaTableLocator.getMetaRegionLocation(this.getZooKeeper()); - if (!metaRegionLocation) { + if (!metaRegionLocation || !regionState.isOpened()) { // Meta location is not verified. It should be in transition, or offline. // We will wait for it to be assigned in enableSSHandWaitForMeta below. assigned++; - if (!rit) { + if (!ConfigUtil.useZKForAssignment(conf)) { + assignMetaZkLess(regionStates, regionState, timeout, previouslyFailedMetaRSs); + } else if (!rit) { + // Assign meta since not already in transition if (currentMetaServer != null) { // If the meta server is not known to be dead or online, @@ -760,6 +770,24 @@ public class HMaster extends HRegionServer implements MasterServices, Server { status.setStatus("META assigned."); } + private void assignMetaZkLess(RegionStates regionStates, RegionState regionState, long timeout, + Set previouslyFailedRs) throws IOException, KeeperException { + ServerName currentServer = regionState.getServerName(); + if (serverManager.isServerOnline(currentServer)) { + LOG.info("Meta was in transition on " + currentServer); + assignmentManager.processRegionInTransitionZkLess(); + } else { + if (currentServer != null) { + splitMetaLogBeforeAssignment(currentServer); + regionStates.logSplit(HRegionInfo.FIRST_META_REGIONINFO); + previouslyFailedRs.add(currentServer); + } + LOG.info("Re-assigning hbase:meta, it was on " + currentServer); + regionStates.updateRegionState(HRegionInfo.FIRST_META_REGIONINFO, State.OFFLINE); + assignmentManager.assignMeta(); + } + } + void initNamespace() throws IOException { //create namespace manager tableNamespaceManager = new TableNamespaceManager(this); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java index fcada083c22..e68a40ce859 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java @@ -41,6 +41,8 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.MultiHConnection; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.zookeeper.KeeperException; import com.google.common.base.Preconditions; @@ -167,12 +169,27 @@ public class RegionStateStore { void updateRegionState(long openSeqNum, RegionState newState, RegionState oldState) { - if (noPersistence || !initialized) { + + if (noPersistence) { return; } - + HRegionInfo hri = newState.getRegion(); - if (!shouldPersistStateChange(hri, newState, oldState)) { + try { + // update meta before checking for initialization. + // meta state stored in zk. + if (hri.isMetaRegion()) { + // persist meta state in MetaTableLocator (which in turn is zk storage currently) + try { + MetaTableLocator.setMetaLocation(server.getZooKeeper(), + newState.getServerName(), newState.getState()); + return; // Done + } catch (KeeperException e) { + throw new IOException("Failed to update meta ZNode", e); + } + } + + if (!initialized || !shouldPersistStateChange(hri, newState, oldState)) { return; } @@ -180,7 +197,6 @@ public class RegionStateStore { ServerName serverName = newState.getServerName(); State state = newState.getState(); - try { int replicaId = hri.getReplicaId(); Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(hri)); StringBuilder info = new StringBuilder("Updating row "); @@ -226,13 +242,13 @@ public class RegionStateStore { } // Called when meta is not on master multiHConnection.processBatchCallback(Arrays.asList(put), TableName.META_TABLE_NAME, null, null); - + } catch (IOException ioe) { LOG.error("Failed to persist region state " + newState, ioe); server.abort("Failed to update region location", ioe); } } - + void splitRegion(HRegionInfo p, HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException { MetaTableAccessor.splitRegion(server.getShortCircuitConnection(), p, a, b, sn); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 69fdb740e56..61b4e65e93c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -95,6 +95,7 @@ import org.apache.hadoop.hbase.ipc.RpcServerInterface; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.procedure.RegionServerProcedureManagerHost; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -1810,7 +1811,7 @@ public class HRegionServer extends HasThread implements // Update ZK, or META if (r.getRegionInfo().isMetaRegion()) { - MetaTableLocator.setMetaLocation(getZooKeeper(), serverName); + MetaTableLocator.setMetaLocation(getZooKeeper(), serverName, State.OPEN); } else if (useZKForAssignment) { MetaTableAccessor.updateRegionLocation(shortCircuitConnection, r.getRegionInfo(), this.serverName, openSeqNum); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java index dccd7caa53b..8a439a82ad4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java @@ -18,13 +18,14 @@ */ package org.apache.hadoop.hbase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import java.io.IOException; import java.net.ConnectException; -import junit.framework.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -32,6 +33,7 @@ import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; @@ -105,6 +107,39 @@ public class TestMetaTableLocator { this.watcher.close(); } + /** + * Test normal operations + */ + @Test public void testMetaLookup() + throws IOException, InterruptedException, ServiceException, KeeperException { + final ClientProtos.ClientService.BlockingInterface client = + Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); + + Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())). + thenReturn(GetResponse.newBuilder().build()); + + final MetaTableLocator mtl = new MetaTableLocator(); + assertNull(mtl.getMetaRegionLocation(this.watcher)); + for (RegionState.State state : RegionState.State.values()) { + if (state.equals(RegionState.State.OPEN)) + continue; + MetaTableLocator.setMetaLocation(this.watcher, SN, state); + assertNull(mtl.getMetaRegionLocation(this.watcher)); + assertEquals(state, MetaTableLocator.getMetaRegionState(this.watcher).getState()); + } + MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); + assertEquals(mtl.getMetaRegionLocation(this.watcher), SN); + assertEquals(RegionState.State.OPEN, + MetaTableLocator.getMetaRegionState(this.watcher).getState()); + + mtl.deleteMetaLocation(this.watcher); + assertNull(MetaTableLocator.getMetaRegionState(this.watcher).getServerName()); + assertEquals(MetaTableLocator.getMetaRegionState(this.watcher).getState(), + RegionState.State.OFFLINE); + assertNull(mtl.getMetaRegionLocation(this.watcher)); + } + + /** * Test interruptable while blocking wait on meta. * @throws IOException @@ -121,7 +156,7 @@ public class TestMetaTableLocator { final MetaTableLocator mtl = new MetaTableLocator(); ServerName meta = new MetaTableLocator().getMetaRegionLocation(this.watcher); - Assert.assertNull(meta); + assertNull(meta); Thread t = new Thread() { @Override public void run() { @@ -153,11 +188,15 @@ public class TestMetaTableLocator { Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())). thenThrow(new ServiceException(ex)); - MetaTableLocator.setMetaLocation(this.watcher, SN); long timeout = UTIL.getConfiguration(). - getLong("hbase.catalog.verification.timeout", 1000); - Assert.assertFalse(new MetaTableLocator().verifyMetaRegionLocation( + getLong("hbase.catalog.verification.timeout", 1000); + MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPENING); + assertFalse(new MetaTableLocator().verifyMetaRegionLocation( connection, watcher, timeout)); + + MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); + assertFalse(new MetaTableLocator().verifyMetaRegionLocation( + connection, watcher, timeout)); } /** @@ -213,9 +252,13 @@ public class TestMetaTableLocator { Mockito.when(connection.getAdmin(Mockito.any(ServerName.class), Mockito.anyBoolean())). thenReturn(implementation); + ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis()); MetaTableLocator.setMetaLocation(this.watcher, - ServerName.valueOf("example.com", 1234, System.currentTimeMillis())); - Assert.assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); + sn, + RegionState.State.OPENING); + assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); + MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN); + assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); } @Test (expected = NotAllMetaRegionsOnlineException.class) @@ -234,22 +277,18 @@ public class TestMetaTableLocator { throws IOException, InterruptedException, KeeperException { final MetaTableLocator mtl = new MetaTableLocator(); ServerName hsa = mtl.getMetaRegionLocation(watcher); - Assert.assertNull(hsa); + assertNull(hsa); // Now test waiting on meta location getting set. Thread t = new WaitOnMetaThread(); startWaitAliveThenWaitItLives(t, 1); // Set a meta location. - hsa = setMetaLocation(); + MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); + hsa = SN; // Join the thread... should exit shortly. t.join(); // Now meta is available. - Assert.assertTrue(mtl.getMetaRegionLocation(watcher).equals(hsa)); - } - - private ServerName setMetaLocation() throws KeeperException { - MetaTableLocator.setMetaLocation(this.watcher, SN); - return SN; + assertTrue(mtl.getMetaRegionLocation(watcher).equals(hsa)); } /** @@ -302,7 +341,7 @@ public class TestMetaTableLocator { } // Wait one second. Threads.sleep(ms); - Assert.assertTrue("Assert " + t.getName() + " still waiting", t.isAlive()); + assertTrue("Assert " + t.getName() + " still waiting", t.isAlive()); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java index 15ccaef700b..8fae42397d5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; @@ -142,12 +143,15 @@ public class TestAssignmentManagerOnCluster { Bytes.toBytes(metaServerName.getServerName())); master.assignmentManager.waitUntilNoRegionsInTransition(60000); } + RegionState metaState = + MetaTableLocator.getMetaRegionState(master.getZooKeeper()); + assertEquals("Meta should be not in transition", + metaState.getState(), RegionState.State.OPEN); assertNotEquals("Meta should be moved off master", metaServerName, master.getServerName()); cluster.killRegionServer(metaServerName); stoppedARegionServer = true; cluster.waitForRegionServerToStop(metaServerName, 60000); - // Wait for SSH to finish final ServerManager serverManager = master.getServerManager(); TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { @@ -160,6 +164,14 @@ public class TestAssignmentManagerOnCluster { // Now, make sure meta is assigned assertTrue("Meta should be assigned", regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO)); + // Now, make sure meta is registered in zk + metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper()); + assertEquals("Meta should be not in transition", + metaState.getState(), RegionState.State.OPEN); + assertEquals("Meta should be assigned", metaState.getServerName(), + regionStates.getRegionServerOfRegion(HRegionInfo.FIRST_META_REGIONINFO)); + assertNotEquals("Meta should be assigned on a different server", + metaState.getServerName(), metaServerName); } finally { if (stoppedARegionServer) { cluster.startRegionServer(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 6a7b952bea1..a1a10fdcaf0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.RequestConverter; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; @@ -67,6 +68,7 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; @@ -1224,7 +1226,8 @@ public class TestMasterFailover { // at this point we only expect 3 regions to be assigned out // (catalogs and namespace, + 1 online region) assertEquals(3, cluster.countServedRegions()); - HRegionInfo hriOnline = onlineTable.getRegionLocation("").getRegionInfo(); + HRegionInfo hriOnline = onlineTable.getRegionLocation( + HConstants.EMPTY_START_ROW).getRegionInfo(); RegionStates regionStates = master.getAssignmentManager().getRegionStates(); RegionStateStore stateStore = master.getAssignmentManager().getRegionStateStore(); @@ -1270,5 +1273,148 @@ public class TestMasterFailover { // Done, shutdown the cluster TEST_UTIL.shutdownMiniCluster(); } + + /** + * Test meta in transition when master failover + */ + @Test(timeout = 180000) + public void testMetaInTransitionWhenMasterFailover() throws Exception { + final int NUM_MASTERS = 1; + final int NUM_RS = 1; + + // Start the cluster + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean("hbase.assignment.usezk", false); + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + log("Cluster started"); + + log("Moving meta off the master"); + HMaster activeMaster = cluster.getMaster(); + HRegionServer rs = cluster.getRegionServer(0); + ServerName metaServerName = cluster.getLiveRegionServerThreads() + .get(0).getRegionServer().getServerName(); + activeMaster.move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + Bytes.toBytes(metaServerName.getServerName())); + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + assertEquals("Meta should be assigned on expected regionserver", + metaServerName, activeMaster.getMetaTableLocator() + .getMetaRegionLocation(activeMaster.getZooKeeper())); + + // Now kill master, meta should remain on rs, where we placed it before. + log("Aborting master"); + activeMaster.abort("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // meta should remain where it was + RegionState metaState = + MetaTableLocator.getMetaRegionState(rs.getZooKeeper()); + assertEquals("hbase:meta should be onlined on RS", + metaState.getServerName(), rs.getServerName()); + assertEquals("hbase:meta should be onlined on RS", + metaState.getState(), State.OPEN); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + // ensure meta is still deployed on RS + metaState = + MetaTableLocator.getMetaRegionState(activeMaster.getZooKeeper()); + assertEquals("hbase:meta should be onlined on RS", + metaState.getServerName(), rs.getServerName()); + assertEquals("hbase:meta should be onlined on RS", + metaState.getState(), State.OPEN); + + // Update meta state as PENDING_OPEN, then kill master + // that simulates, that RS successfully deployed, but + // RPC was lost right before failure. + // region server should expire (how it can be verified?) + MetaTableLocator.setMetaLocation(activeMaster.getZooKeeper(), + rs.getServerName(), State.PENDING_OPEN); + HRegion meta = rs.getFromOnlineRegions(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); + rs.removeFromOnlineRegions(meta, null); + meta.close(); + + log("Aborting master"); + activeMaster.abort("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + log("Meta was assigned"); + + metaState = + MetaTableLocator.getMetaRegionState(activeMaster.getZooKeeper()); + assertEquals("hbase:meta should be onlined on RS", + metaState.getServerName(), rs.getServerName()); + assertEquals("hbase:meta should be onlined on RS", + metaState.getState(), State.OPEN); + + // Update meta state as PENDING_CLOSE, then kill master + // that simulates, that RS successfully deployed, but + // RPC was lost right before failure. + // region server should expire (how it can be verified?) + MetaTableLocator.setMetaLocation(activeMaster.getZooKeeper(), + rs.getServerName(), State.PENDING_CLOSE); + + log("Aborting master"); + activeMaster.abort("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + rs.getRSRpcServices().closeRegion(null, RequestConverter.buildCloseRegionRequest( + rs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), false)); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + log("Meta was assigned"); + + rs.getRSRpcServices().closeRegion( + null, + RequestConverter.buildCloseRegionRequest(rs.getServerName(), + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), false)); + + // Set a dummy server to check if master reassigns meta on restart + MetaTableLocator.setMetaLocation(activeMaster.getZooKeeper(), + ServerName.valueOf("dummyserver.example.org", 1234, -1L), State.OPEN); + + log("Aborting master"); + activeMaster.stop("test-kill"); + + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + log("Meta was assigned"); + + // Done, shutdown the cluster + TEST_UTIL.shutdownMiniCluster(); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index 00982b062b6..de7b8f811cc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -39,12 +39,12 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MetaMockingUtil; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.MetaMockingUtil; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; @@ -158,7 +158,8 @@ public class TestMasterNoCluster { final MockRegionServer rs2 = new MockRegionServer(conf, sn2); // Put some data into the servers. Make it look like sn0 has the metaH // Put data into sn2 so it looks like it has a few regions for a table named 't'. - MetaTableLocator.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName()); + MetaTableLocator.setMetaLocation(rs0.getZooKeeper(), + rs0.getServerName(), RegionState.State.OPEN); final TableName tableName = TableName.valueOf("t"); Result [] results = new Result [] { MetaMockingUtil.getMetaTableRowResult(