Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-1066 Master should support close/open/reassignment/enable/disable operations on individual regions 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@729168 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2008-12-23 23:10:25 +00:00
parent e1d9bf64c2
commit d47bdad4fd
12 changed files with 263 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -185,6 +185,8 @@ Release 0.19.0 - Unreleased
HBASE-1053 bring recent rpc changes down from hadoop
HBASE-1056 [migration] enable blockcaching on .META. table
HBASE-1069 Show whether HRegion major compacts or not in INFO level
HBASE-1066 Master should support close/open/reassignment/enable/disable
operations on individual regions
NEW FEATURES
HBASE-875 Use MurmurHash instead of JenkinsHash [in bloomfilters]

33
bin/HBase.rb
View File

@ -21,6 +21,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor
import org.apache.hadoop.hbase.HTableDescriptor
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.util.Writables
import org.apache.hadoop.hbase.HRegionInfo
module HBase
COLUMN = "COLUMN"
@ -93,6 +94,29 @@ module HBase
@formatter.footer(now)
end
def enable_region(regionName)
online(regionName, false)
end
def disable_region(regionName)
online(regionName, true)
end
def online(regionName, onOrOff)
now = Time.now
meta = HTable.new(HConstants::META_TABLE_NAME)
bytes = Bytes.toBytes(regionName)
hriBytes = meta.get(bytes, HConstants::COL_REGIONINFO).getValue()
hri = Writables.getWritable(hriBytes, HRegionInfo.new());
hri.setOffline(onOrOff)
p hri
bu = BatchUpdate.new(bytes)
bu.put(HConstants::COL_REGIONINFO, Writables.getBytes(hri))
meta.commit(bu);
@formatter.header()
@formatter.footer(now)
end
def drop(tableName)
now = Time.now
@formatter.header()
@ -164,6 +188,15 @@ module HBase
@formatter.footer(now)
end
def close_region(regionName, server)
now = Time.now
s = nil
s = [server].to_java if server
@admin.closeRegion(regionName, s)
@formatter.header()
@formatter.footer(now)
end
# Make a legal column name of the passed String
# Check string ends in colon. If not, add it.
def makeColumnName(arg)

35
bin/hirb.rb
View File

@ -101,6 +101,27 @@ promoteConstants(HBase.constants)
# General shell methods
def tools
# Help for hbase shell surgery tools
h = <<HERE
HBASE SURGERY TOOLS:
enable_region Enable a single region. For example,
hbase> enable_region 'REGIONNAME'
disable_region Disable a single region
close_region Close a single region. Optionally specify regionserver.
Examples:
hbase> close_region 'REGIONNAME'
hbase> close_region 'REGIONNAME', 'REGIONSERVER_IP:PORT'
Above commands are for 'experts'-only as misuse can damage an install
HERE
puts h
end
def help
# Output help. Help used to be a dictionary of name to short and long
# descriptions emitted using Formatters but awkward getting it to show
@ -179,6 +200,8 @@ HBASE SHELL COMMANDS:
hbase> put 't1', 'r1', 'c1', 'value', ts1
tools Listing of hbase surgery tools
scan Scan a table; pass table name and optionally a dictionary of scanner
specifications. Scanner specifications may include one or more of
the following: LIMIT, STARTROW, STOPROW, TIMESTAMP, or COLUMNS. If
@ -263,6 +286,14 @@ def disable(table)
admin().disable(table)
end
def enable_region(regionName)
admin().enable_region(regionName)
end
def disable_region(regionName)
admin().disable_region(regionName)
end
def exists(table)
admin().exists(table)
end
@ -270,6 +301,10 @@ end
def truncate(table)
admin().truncate(table)
end
def close_region(regionName, server = nil)
admin().close_region(regionName, server)
end
# CRUD

4
src/java/org/apache/hadoop/hbase/HConstants.java
View File

@ -265,5 +265,7 @@ public interface HConstants {
public static final int MODIFY_TABLE_SPLIT = 2;
/** modifyTable op for forcing a compaction */
public static final int MODIFY_TABLE_COMPACT = 3;
// Messages client can send master.
public static final int MODIFY_CLOSE_REGION = MODIFY_TABLE_COMPACT + 1;
}

2
src/java/org/apache/hadoop/hbase/HTableDescriptor.java
View File

@ -165,7 +165,7 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
public HTableDescriptor(final byte [] name) {
super();
setMetaFlags(this.name);
this.name = this.isMetaRegion() ? name: isLegalTableName(name);
this.name = this.isMetaRegion()? name: isLegalTableName(name);
this.nameAsString = Bytes.toString(this.name);
}

84
src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
View File

@ -40,7 +40,9 @@ import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.MetaUtils;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.ipc.RemoteException;
@ -505,6 +507,39 @@ public class HBaseAdmin {
}
}
/**
* Close a region. For expert-admins.
* @param regionname
* @param args Optional server name. Otherwise, we'll send close to the
* server registered in .META.
* @throws IOException
*/
public void closeRegion(final String regionname, final Object... args)
throws IOException {
closeRegion(Bytes.toBytes(regionname), args);
}
/**
* Close a region. For expert-admins.
* @param regionname
* @param args Optional server name. Otherwise, we'll send close to the
* server registered in .META.
* @throws IOException
*/
public void closeRegion(final byte [] regionname, final Object... args)
throws IOException {
// Be careful. Must match the handler over in HMaster at MODIFY_CLOSE_REGION
int len = (args == null)? 0: args.length;
int xtraArgsCount = 1;
Object [] newargs = new Object[len + xtraArgsCount];
newargs[0] = regionname;
for (int i = 0; i < len; i++) {
newargs[i + xtraArgsCount] = args[i];
}
modifyTable(HConstants.META_TABLE_NAME, HConstants.MODIFY_CLOSE_REGION,
newargs);
}
/**
* Modify an existing table
*
@ -518,20 +553,26 @@ public class HBaseAdmin {
if (this.master == null) {
throw new MasterNotRunningException("master has been shut down");
}
HTableDescriptor.isLegalTableName(tableName);
// Let pass if its a catalog table. Used by admins.
if (!MetaUtils.isMetaTableName(tableName)) {
// This will throw exception
HTableDescriptor.isLegalTableName(tableName);
}
Writable[] arr = null;
try {
switch (op) {
case HConstants.MODIFY_TABLE_SET_HTD: {
case HConstants.MODIFY_TABLE_SET_HTD:
if (args == null || args.length < 1 ||
!(args[0] instanceof HTableDescriptor))
throw new IOException("SET_HTD requires a HTableDescriptor");
Writable[] arr = new Writable[1];
!(args[0] instanceof HTableDescriptor)) {
throw new IllegalArgumentException("SET_HTD requires a HTableDescriptor");
}
arr = new Writable[1];
arr[0] = (HTableDescriptor)args[0];
this.master.modifyTable(tableName, op, arr);
} break;
break;
case HConstants.MODIFY_TABLE_COMPACT:
case HConstants.MODIFY_TABLE_SPLIT: {
Writable[] arr = null;
case HConstants.MODIFY_TABLE_SPLIT:
if (args != null && args.length > 0) {
arr = new Writable[1];
if (args[0] instanceof byte[]) {
@ -539,12 +580,35 @@ public class HBaseAdmin {
} else if (args[0] instanceof ImmutableBytesWritable) {
arr[0] = (ImmutableBytesWritable)args[0];
} else {
throw new IOException("SPLIT or COMPACT with arg requires byte[] or ImmutableBytesWritable");
throw new IllegalArgumentException("SPLIT or COMPACT with arg " +
"requires byte[] or ImmutableBytesWritable");
}
}
this.master.modifyTable(tableName, op, arr);
break;
}
case HConstants.MODIFY_CLOSE_REGION:
if (args == null || args.length < 1) {
throw new IllegalArgumentException("Requires at least a region name");
}
arr = new Writable[args.length];
for (int i = 0; i < args.length; i++) {
if (args[i] instanceof byte[]) {
arr[i] = new ImmutableBytesWritable((byte[])args[i]);
} else if (args[i] instanceof ImmutableBytesWritable) {
arr[i] = (ImmutableBytesWritable)args[i];
} else if (args[i] instanceof String) {
arr[i] = new ImmutableBytesWritable(Bytes.toBytes((String)args[i]));
} else if (args[i] instanceof Boolean) {
arr[i] = new BooleanWritable(((Boolean)args[i]).booleanValue());
} else {
throw new IllegalArgumentException("Requires byte [] or " +
"ImmutableBytesWritable, not " + args[i]);
}
}
this.master.modifyTable(tableName, op, arr);
break;
default:
throw new IOException("unknown modifyTable op " + op);
}

4
src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
View File

@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.MetaUtils;
import org.apache.hadoop.hbase.util.SoftValueSortedMap;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.ipc.RemoteException;
@ -270,8 +271,7 @@ public class HConnectionManager implements HConstants {
* of a catalog table.
*/
private static boolean isMetaTableName(final byte [] n) {
return Bytes.equals(n, ROOT_TABLE_NAME) ||
Bytes.equals(n, META_TABLE_NAME);
return MetaUtils.isMetaTableName(n);
}
public HRegionLocation getRegionLocation(final byte [] name,

11
src/java/org/apache/hadoop/hbase/master/ChangeTableState.java
View File

@ -68,7 +68,6 @@ class ChangeTableState extends TableOperation {
protected void postProcessMeta(MetaRegion m, HRegionInterface server)
throws IOException {
// Process regions not being served
if (LOG.isDebugEnabled()) {
LOG.debug("processing unserved regions");
}
@ -80,20 +79,15 @@ class ChangeTableState extends TableOperation {
}
continue;
}
// Update meta table
if (LOG.isDebugEnabled()) {
LOG.debug("updating columns in row: " + i.getRegionNameAsString());
}
// Update meta table
BatchUpdate b = new BatchUpdate(i.getRegionName());
updateRegionInfo(b, i);
b.delete(COL_SERVER);
b.delete(COL_STARTCODE);
server.batchUpdate(m.getRegionName(), b, -1L);
if (LOG.isDebugEnabled()) {
LOG.debug("updated columns in row: " + i.getRegionNameAsString());
LOG.debug("Updated columns in row: " + i.getRegionNameAsString());
}
if (online) {
@ -109,7 +103,6 @@ class ChangeTableState extends TableOperation {
}
// Process regions currently being served
if (LOG.isDebugEnabled()) {
LOG.debug("processing regions currently being served");
}

59
src/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -36,8 +36,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
@ -61,12 +59,12 @@ import org.apache.hadoop.hbase.client.ServerConnectionManager;
import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
import org.apache.hadoop.hbase.ipc.HBaseServer;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
@ -75,6 +73,8 @@ import org.apache.hadoop.hbase.util.InfoServer;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Sleeper;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@ -756,6 +756,31 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
return null;
}
/**
* Get row from meta table.
* @param row
* @param columns
* @return RowResult
* @throws IOException
*/
protected RowResult getFromMETA(final byte [] row, final byte [][] columns)
throws IOException {
MetaRegion meta = this.regionManager.getMetaRegionForRow(row);
HRegionInterface srvr = getMETAServer(meta);
return srvr.getRow(meta.getRegionName(), row, columns,
HConstants.LATEST_TIMESTAMP, 1, -1);
}
/*
* @param meta
* @return Server connection to <code>meta</code> .META. region.
* @throws IOException
*/
private HRegionInterface getMETAServer(final MetaRegion meta)
throws IOException {
return this.connection.getHRegionConnection(meta.getServer());
}
public void modifyTable(final byte[] tableName, int op, Writable[] args)
throws IOException {
switch (op) {
@ -786,6 +811,34 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
pair.getFirst(), pair.getSecond(), op);
}
break;
case MODIFY_CLOSE_REGION:
if (args == null || args.length < 1 || args.length > 2) {
throw new IOException("Requires at least a region name; " +
"or cannot have more than region name and servername");
}
// Arguments are regionname and an optional server name.
byte [] regionname = ((ImmutableBytesWritable)args[0]).get();
String servername = null;
if (args.length == 2) {
servername = Bytes.toString(((ImmutableBytesWritable)args[1]).get());
}
// Need hri
RowResult rr = getFromMETA(regionname, HConstants.COLUMN_FAMILY_ARRAY);
HRegionInfo hri = getHRegionInfo(rr.getRow(), rr);
if (servername == null) {
// Get server from the .META. if it wasn't passed as argument
servername = Writables.cellToString(rr.get(COL_SERVER));
}
LOG.info("Marking " + hri.getRegionNameAsString() +
" as closed on " + servername + "; cleaning SERVER + STARTCODE; " +
"master will tell regionserver to close region on next heartbeat");
this.regionManager.markToClose(servername, hri);
MetaRegion meta = this.regionManager.getMetaRegionForRow(regionname);
HRegionInterface srvr = getMETAServer(meta);
HRegion.cleanRegionInMETA(srvr, meta.getRegionName(), hri);
break;
default:
throw new IOException("unsupported modifyTable op " + op);
}

41
src/java/org/apache/hadoop/hbase/master/RegionManager.java
View File

@ -29,12 +29,14 @@ import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.Collections;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -73,9 +75,8 @@ class RegionManager implements HConstants {
private final AtomicInteger numberOfMetaRegions = new AtomicInteger();
/** These are the online meta regions */
private final SortedMap<byte [], MetaRegion> onlineMetaRegions =
Collections.synchronizedSortedMap(new TreeMap<byte [],
MetaRegion>(Bytes.BYTES_COMPARATOR));
private final NavigableMap<byte [], MetaRegion> onlineMetaRegions =
new ConcurrentSkipListMap<byte [], MetaRegion>(Bytes.BYTES_COMPARATOR);
private static final byte[] OVERLOADED = Bytes.toBytes("Overloaded");
@ -542,7 +543,6 @@ class RegionManager implements HConstants {
throws NotAllMetaRegionsOnlineException {
byte [] firstMetaRegion = null;
Set<MetaRegion> metaRegions = new HashSet<MetaRegion>();
if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
if (rootRegionLocation.get() == null) {
throw new NotAllMetaRegionsOnlineException(
@ -550,7 +550,6 @@ class RegionManager implements HConstants {
}
metaRegions.add(new MetaRegion(rootRegionLocation.get(),
HRegionInfo.ROOT_REGIONINFO.getRegionName()));
} else {
if (!areAllMetaRegionsOnline()) {
throw new NotAllMetaRegionsOnlineException();
@ -568,7 +567,21 @@ class RegionManager implements HConstants {
}
return metaRegions;
}
/**
* Get metaregion that would host passed in row.
* @param row Row need to know all the meta regions for
* @return set of MetaRegion objects that contain the table
* @throws NotAllMetaRegionsOnlineException
*/
public MetaRegion getMetaRegionForRow(final byte [] row)
throws NotAllMetaRegionsOnlineException {
if (!areAllMetaRegionsOnline()) {
throw new NotAllMetaRegionsOnlineException();
}
return this.onlineMetaRegions.floorEntry(row).getValue();
}
/**
* Create a new HRegion, put a row for it into META (or ROOT), and mark the
* new region unassigned so that it will get assigned to a region server.
@ -724,13 +737,11 @@ class RegionManager implements HConstants {
* @param serverName address info of server
* @param info region to close
*/
public void markToClose(String serverName, HRegionInfo info) {
synchronized (regionsToClose) {
Map<byte [], HRegionInfo> serverToClose = regionsToClose.get(serverName);
if (serverToClose != null) {
serverToClose.put(info.getRegionName(), info);
}
}
public void markToClose(final String serverName, final HRegionInfo info) {
Map<byte [], HRegionInfo> toclose =
new TreeMap<byte [], HRegionInfo>(Bytes.BYTES_COMPARATOR);
toclose.put(info.getRegionName(), info);
markToCloseBulk(serverName, toclose);
}
/**
@ -738,8 +749,8 @@ class RegionManager implements HConstants {
* @param serverName address info of server
* @param map map of region names to region infos of regions to close
*/
public void markToCloseBulk(String serverName,
Map<byte [], HRegionInfo> map) {
public void markToCloseBulk(final String serverName,
final Map<byte [], HRegionInfo> map) {
synchronized (regionsToClose) {
Map<byte [], HRegionInfo> regions = regionsToClose.get(serverName);
if (regions != null) {

19
src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -2157,6 +2157,25 @@ public class HRegion implements HConstants {
// server.
srvr.batchUpdate(metaRegionName, b, -1L);
}
/**
* Clean COL_SERVER and COL_STARTCODE for passed <code>info</code> in
* <code>.META.</code>
* @param srvr
* @param metaRegionName
* @param info
* @throws IOException
*/
public static void cleanRegionInMETA(final HRegionInterface srvr,
final byte [] metaRegionName, final HRegionInfo info)
throws IOException {
BatchUpdate b = new BatchUpdate(info.getRegionName());
b.delete(COL_SERVER);
b.delete(COL_STARTCODE);
// If carrying splits, they'll be in place when we show up on new
// server.
srvr.batchUpdate(metaRegionName, b, LATEST_TIMESTAMP);
}
/**
* Deletes all the files for a HRegion

11
src/java/org/apache/hadoop/hbase/util/MetaUtils.java
View File

@ -442,4 +442,13 @@ public class MetaUtils {
}});
return result;
}
}
/**
* @param n Table name.
* @return True if a catalog table, -ROOT- or .META.
*/
public static boolean isMetaTableName(final byte [] n) {
return Bytes.equals(n, HConstants.ROOT_TABLE_NAME) ||
Bytes.equals(n, HConstants.META_TABLE_NAME);
}
}