Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-1832 Faster enable/disable/delete 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@822024 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2009-10-05 21:25:09 +00:00
parent 72adc4c9e2
commit 7130baf116
11 changed files with 112 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -94,6 +94,7 @@ Release 0.21.0 - Unreleased
away from the HBase cluster (Andrei Dragomir via Stack)
HBASE-1879 ReadOnly transactions generate WAL activity (Clint Morgan via Stack)
HBASE-1875 Compression test utility
HBASE-1832 Faster enable/disable/delete
OPTIMIZATIONS

28
src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
View File

@ -330,16 +330,17 @@ public class HBaseAdmin {
}
// Wait until all regions are enabled
for (int tries = 0;
(tries < numRetries) && (!isTableEnabled(tableName));
tries++) {
boolean enabled = false;
for (int tries = 0; tries < this.numRetries; tries++) {
enabled = isTableEnabled(tableName);
if (enabled) break;
long sleep = getPauseTime(tries);
if (LOG.isDebugEnabled()) {
LOG.debug("Sleep. Waiting for all regions to be enabled from " +
Bytes.toString(tableName));
LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " +
"enabled in " + Bytes.toString(tableName));
}
try {
Thread.sleep(getPauseTime(tries));
Thread.sleep(sleep);
} catch (InterruptedException e) {
// continue
}
@ -348,8 +349,8 @@ public class HBaseAdmin {
Bytes.toString(tableName));
}
}
if (!isTableEnabled(tableName))
throw new IOException("unable to enable table " +
if (!enabled)
throw new IOException("Unable to enable table " +
Bytes.toString(tableName));
LOG.info("Enabled table " + Bytes.toString(tableName));
}
@ -385,9 +386,10 @@ public class HBaseAdmin {
}
// Wait until all regions are disabled
for (int tries = 0;
(tries < numRetries) && (!isTableDisabled(tableName));
tries++) {
boolean disabled = false;
for (int tries = 0; tries < this.numRetries; tries++) {
disabled = isTableDisabled(tableName);
if (disabled) break;
if (LOG.isDebugEnabled()) {
LOG.debug("Sleep. Waiting for all regions to be disabled from " +
Bytes.toString(tableName));
@ -402,7 +404,7 @@ public class HBaseAdmin {
Bytes.toString(tableName));
}
}
if (!isTableDisabled(tableName)) {
if (!disabled) {
throw new RegionException("Retries exhausted, it took too long to wait"+
" for the table " + Bytes.toString(tableName) + " to be disabled.");
}

36
src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
View File

@ -450,8 +450,8 @@ public class HConnectionManager implements HConstants {
* Returns true if all regions are offline
* Returns false in any other case
*/
private boolean testTableOnlineState(byte[] tableName,
boolean online) throws IOException {
private boolean testTableOnlineState(byte[] tableName, boolean online)
throws IOException {
if (!tableExists(tableName)) {
throw new TableNotFoundException(Bytes.toString(tableName));
}
@ -459,7 +459,6 @@ public class HConnectionManager implements HConstants {
// The root region is always enabled
return true;
}
int rowsScanned = 0;
int rowsOffline = 0;
byte[] startKey =
@ -468,6 +467,8 @@ public class HConnectionManager implements HConstants {
HRegionInfo currentRegion = null;
Scan scan = new Scan(startKey);
scan.addColumn(CATALOG_FAMILY, REGIONINFO_QUALIFIER);
int rows = this.conf.getInt("hbase.meta.scanner.caching", 100);
scan.setCaching(rows);
ScannerCallable s = new ScannerCallable(this,
(Bytes.equals(tableName, HConstants.META_TABLE_NAME) ?
HConstants.ROOT_TABLE_NAME : HConstants.META_TABLE_NAME), scan);
@ -482,10 +483,14 @@ public class HConnectionManager implements HConstants {
currentRegion = s.getHRegionInfo();
Result r = null;
Result [] rrs = null;
while ((rrs = getRegionServerWithRetries(s)) != null) {
do {
rrs = getRegionServerWithRetries(s);
if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
break; //exit completely
}
r = rrs[0];
byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER);
byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER);
if (value != null) {
HRegionInfo info = Writables.getHRegionInfoOrNull(value);
if (info != null) {
@ -495,20 +500,21 @@ public class HConnectionManager implements HConstants {
}
}
}
}
} while(true);
endKey = currentRegion.getEndKey();
} while (!(endKey == null ||
} while (!(endKey == null ||
Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)));
}
finally {
} finally {
s.setClose();
// Doing below will call 'next' again and this will close the scanner
// Without it we leave scanners open.
getRegionServerWithRetries(s);
}
boolean onlineOffline =
online ? rowsOffline == 0 : rowsOffline == rowsScanned;
return rowsScanned > 0 && onlineOffline;
LOG.debug("Rowscanned=" + rowsScanned + ", rowsOffline=" + rowsOffline);
boolean onOffLine = online? rowsOffline == 0: rowsOffline == rowsScanned;
return rowsScanned > 0 && onOffLine;
}
private static class HTableDescriptorFinder
implements MetaScanner.MetaScannerVisitor {
byte[] tableName;

26
src/java/org/apache/hadoop/hbase/client/ScannerCallable.java
View File

@ -94,20 +94,20 @@ public class ScannerCallable extends ServerCallable<Result[]> {
}
private void close() {
if (this.scannerId == -1L) {
return;
}
try {
this.server.close(this.scannerId);
} catch (IOException e) {
Log.warn("Ignore, probably already closed", e);
}
this.scannerId = -1L;
if (this.scannerId == -1L) {
return;
}
try {
this.server.close(this.scannerId);
} catch (IOException e) {
Log.warn("Ignore, probably already closed", e);
}
this.scannerId = -1L;
}
protected long openScanner() throws IOException {
return server.openScanner(
this.location.getRegionInfo().getRegionName(), scan);
return this.server.openScanner(this.location.getRegionInfo().getRegionName(),
this.scan);
}
protected Scan getScan() {
@ -118,7 +118,7 @@ public class ScannerCallable extends ServerCallable<Result[]> {
* Call this when the next invocation of call should close the scanner
*/
public void setClose() {
closed = true;
this.closed = true;
}
/**
@ -146,4 +146,4 @@ public class ScannerCallable extends ServerCallable<Result[]> {
public void setCaching(int caching) {
this.caching = caching;
}
}
}

51
src/java/org/apache/hadoop/hbase/master/ChangeTableState.java
View File

@ -20,9 +20,9 @@
package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -32,14 +32,15 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Writables;
/** Instantiated to enable or disable a table */
/**
* Instantiated to enable or disable a table
*/
class ChangeTableState extends TableOperation {
private final Log LOG = LogFactory.getLog(this.getClass());
private boolean online;
// Do in order.
protected final Map<String, HashSet<HRegionInfo>> servedRegions =
new HashMap<String, HashSet<HRegionInfo>>();
new TreeMap<String, HashSet<HRegionInfo>>();
protected long lockid;
ChangeTableState(final HMaster master, final byte [] tableName,
@ -51,14 +52,13 @@ class ChangeTableState extends TableOperation {
@Override
protected void processScanItem(String serverName, HRegionInfo info) {
if (isBeingServed(serverName)) {
HashSet<HRegionInfo> regions = servedRegions.get(serverName);
HashSet<HRegionInfo> regions = this.servedRegions.get(serverName);
if (regions == null) {
regions = new HashSet<HRegionInfo>();
}
regions.add(info);
servedRegions.put(serverName, regions);
this.servedRegions.put(serverName, regions);
}
}
@ -67,13 +67,13 @@ class ChangeTableState extends TableOperation {
throws IOException {
// Process regions not being served
if (LOG.isDebugEnabled()) {
LOG.debug("processing unserved regions");
LOG.debug("Processing unserved regions");
}
for (HRegionInfo i: unservedRegions) {
for (HRegionInfo i: this.unservedRegions) {
if (i.isOffline() && i.isSplit()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping region " + i.toString() +
" because it is offline because it has been split");
" because it is offline and split");
}
continue;
}
@ -81,36 +81,36 @@ class ChangeTableState extends TableOperation {
// Update meta table
Put put = updateRegionInfo(i);
server.put(m.getRegionName(), put);
Delete delete = new Delete(i.getRegionName());
delete.deleteColumns(CATALOG_FAMILY, SERVER_QUALIFIER);
delete.deleteColumns(CATALOG_FAMILY, STARTCODE_QUALIFIER);
server.delete(m.getRegionName(), delete);
if (LOG.isDebugEnabled()) {
LOG.debug("Updated columns in row: " + i.getRegionNameAsString());
LOG.debug("Removed server and startcode from row and set online=" +
this.online + ": " + i.getRegionNameAsString());
}
synchronized (master.regionManager) {
if (online) {
if (this.online) {
// Bring offline regions on-line
if (!master.regionManager.regionIsOpening(i.getRegionNameAsString())) {
master.regionManager.setUnassigned(i, false);
if (!this.master.regionManager.regionIsOpening(i.getRegionNameAsString())) {
this.master.regionManager.setUnassigned(i, false);
}
} else {
// Prevent region from getting assigned.
master.regionManager.removeRegion(i);
this.master.regionManager.removeRegion(i);
}
}
}
// Process regions currently being served
if (LOG.isDebugEnabled()) {
LOG.debug("processing regions currently being served");
LOG.debug("Processing regions currently being served");
}
synchronized (master.regionManager) {
for (Map.Entry<String, HashSet<HRegionInfo>> e: servedRegions.entrySet()) {
synchronized (this.master.regionManager) {
for (Map.Entry<String, HashSet<HRegionInfo>> e:
this.servedRegions.entrySet()) {
String serverName = e.getKey();
if (online) {
if (this.online) {
LOG.debug("Already online");
continue; // Already being served
}
@ -118,14 +118,15 @@ class ChangeTableState extends TableOperation {
// Cause regions being served to be taken off-line and disabled
for (HRegionInfo i: e.getValue()) {
if (LOG.isDebugEnabled()) {
LOG.debug("adding region " + i.getRegionNameAsString() + " to kill list");
LOG.debug("Adding region " + i.getRegionNameAsString() +
" to setClosing list");
}
// this marks the regions to be closed
master.regionManager.setClosing(serverName, i, true);
this.master.regionManager.setClosing(serverName, i, true);
}
}
}
servedRegions.clear();
this.servedRegions.clear();
}
protected Put updateRegionInfo(final HRegionInfo i)

6
src/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -844,8 +844,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
Scan scan = new Scan(firstRowInTable);
scan.addColumn(CATALOG_FAMILY, REGIONINFO_QUALIFIER);
scan.addColumn(CATALOG_FAMILY, SERVER_QUALIFIER);
long scannerid =
srvr.openScanner(metaRegionName, scan);
long scannerid = srvr.openScanner(metaRegionName, scan);
try {
while (true) {
Result data = srvr.next(scannerid);
@ -882,8 +881,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
Scan scan = new Scan(firstRowInTable);
scan.addColumn(CATALOG_FAMILY, REGIONINFO_QUALIFIER);
scan.addColumn(CATALOG_FAMILY, SERVER_QUALIFIER);
long scannerid =
srvr.openScanner(metaRegionName, scan);
long scannerid = srvr.openScanner(metaRegionName, scan);
try {
while (true) {
Result data = srvr.next(scannerid);

27
src/java/org/apache/hadoop/hbase/master/RegionManager.java
View File

@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.HServerLoad;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.regionserver.HLog;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@ -511,25 +512,30 @@ class RegionManager implements HConstants {
LOG.info("Skipped " + skipped + " region(s) that are in transition states");
}
/*
* PathFilter that accepts hbase tables only.
*/
static class TableDirFilter implements PathFilter {
public boolean accept(Path path) {
// skip the region servers' log dirs && version file
// HBASE-1112 want to sperate the log dirs from table's data dirs by a special character.
// HBASE-1112 want to separate the log dirs from table's data dirs by a
// special character.
String pathname = path.getName();
return !pathname.startsWith("log_") && !pathname.equals(VERSION_FILE_NAME);
return !pathname.equals(HLog.HREGION_LOGDIR_NAME) &&
!pathname.equals(VERSION_FILE_NAME);
}
}
static class RegionDirFilter implements PathFilter {
/*
* PathFilter that accepts all but compaction.dir names.
*/
static class RegionDirFilter implements PathFilter {
public boolean accept(Path path) {
return !path.getName().equals(HREGION_COMPACTIONDIR_NAME);
}
}
/**
* @return the rough number of the regions on fs
* Note: this method simply counts the regions on fs by accumulating all the dirs
@ -538,10 +544,8 @@ class RegionManager implements HConstants {
*/
public int countRegionsOnFS() throws IOException {
int regions = 0;
FileStatus[] tableDirs =
FileStatus[] tableDirs =
master.fs.listStatus(master.rootdir, new TableDirFilter());
FileStatus[] regionDirs;
RegionDirFilter rdf = new RegionDirFilter();
for(FileStatus tabledir : tableDirs) {
@ -550,7 +554,6 @@ class RegionManager implements HConstants {
regions += regionDirs.length;
}
}
return regions;
}
@ -1650,4 +1653,4 @@ class RegionManager implements HConstants {
return Bytes.compareTo(getRegionName(), o.getRegionName());
}
}
}
}

11
src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java
View File

@ -56,12 +56,13 @@ abstract class RetryableMetaOperation<T> implements Callable<T> {
protected T doWithRetries()
throws IOException, RuntimeException {
List<IOException> exceptions = new ArrayList<IOException>();
for(int tries = 0; tries < master.numRetries; tries++) {
if (master.closed.get()) {
for (int tries = 0; tries < this.master.numRetries; tries++) {
if (this.master.closed.get()) {
return null;
}
try {
this.server = master.connection.getHRegionConnection(m.getServer());
this.server =
this.master.connection.getHRegionConnection(m.getServer());
return this.call();
} catch (IOException e) {
if (e instanceof TableNotFoundException ||
@ -94,8 +95,8 @@ abstract class RetryableMetaOperation<T> implements Callable<T> {
LOG.debug("Exception in RetryableMetaOperation: ", e);
throw new RuntimeException(e);
}
sleeper.sleep();
this.sleeper.sleep();
}
return null;
}
}
}

2
src/java/org/apache/hadoop/hbase/master/ServerManager.java
View File

@ -477,7 +477,7 @@ class ServerManager implements HConstants {
// Should we tell it close regions because its overloaded? If its
// currently opening regions, leave it alone till all are open.
if ((openingCount < this.nobalancingCount)) {
if (openingCount < this.nobalancingCount) {
this.master.regionManager.assignRegions(serverInfo, mostLoadedRegions,
returnMsgs);
}

20
src/java/org/apache/hadoop/hbase/master/TableOperation.java
View File

@ -21,9 +21,9 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.util.Bytes;
abstract class TableOperation implements HConstants {
private final Set<MetaRegion> metaRegions;
protected final byte [] tableName;
protected final Set<HRegionInfo> unservedRegions = new HashSet<HRegionInfo>();
// Do regions in order.
protected final Set<HRegionInfo> unservedRegions = new TreeSet<HRegionInfo>();
protected HMaster master;
protected TableOperation(final HMaster master, final byte [] tableName)
@ -79,15 +80,17 @@ abstract class TableOperation implements HConstants {
// Open a scanner on the meta region
byte [] tableNameMetaStart =
Bytes.toBytes(Bytes.toString(tableName) + ",,");
Bytes.toBytes(Bytes.toString(tableName) + ",,");
Scan scan = new Scan(tableNameMetaStart).addFamily(CATALOG_FAMILY);
long scannerId = server.openScanner(m.getRegionName(), scan);
long scannerId = this.server.openScanner(m.getRegionName(), scan);
int rows = this.master.getConfiguration().
getInt("hbase.meta.scanner.caching", 100);
scan.setCaching(rows);
List<byte []> emptyRows = new ArrayList<byte []>();
try {
while (true) {
Result values = server.next(scannerId);
if(values == null || values.isEmpty()) {
Result values = this.server.next(scannerId);
if (values == null || values.isEmpty()) {
break;
}
HRegionInfo info = this.master.getHRegionInfo(values.getRow(), values);
@ -119,7 +122,7 @@ abstract class TableOperation implements HConstants {
} finally {
if (scannerId != -1L) {
try {
server.close(scannerId);
this.server.close(scannerId);
} catch (IOException e) {
e = RemoteExceptionHandler.checkIOException(e);
LOG.error("closing scanner", e);
@ -143,7 +146,6 @@ abstract class TableOperation implements HConstants {
postProcessMeta(m, server);
unservedRegions.clear();
return Boolean.TRUE;
}
}

4
src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -1888,9 +1888,7 @@ public class HRegionServer implements HConstants, HRegionInterface,
requestCount.incrementAndGet();
try {
HRegion r = getRegion(regionName);
InternalScanner s = r.getScanner(scan);
long scannerId = addScanner(s);
return scannerId;
return addScanner(r.getScanner(scan));
} catch (Throwable t) {
throw convertThrowableToIOE(cleanup(t, "Failed openScanner"));
}