diff --git a/CHANGES.txt b/CHANGES.txt index 74df819a6fd..38a027bf02a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -94,6 +94,7 @@ Release 0.21.0 - Unreleased away from the HBase cluster (Andrei Dragomir via Stack) HBASE-1879 ReadOnly transactions generate WAL activity (Clint Morgan via Stack) HBASE-1875 Compression test utility + HBASE-1832 Faster enable/disable/delete OPTIMIZATIONS diff --git a/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 68474d2a051..9656b43bbd8 100644 --- a/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -330,16 +330,17 @@ public class HBaseAdmin { } // Wait until all regions are enabled - - for (int tries = 0; - (tries < numRetries) && (!isTableEnabled(tableName)); - tries++) { + boolean enabled = false; + for (int tries = 0; tries < this.numRetries; tries++) { + enabled = isTableEnabled(tableName); + if (enabled) break; + long sleep = getPauseTime(tries); if (LOG.isDebugEnabled()) { - LOG.debug("Sleep. Waiting for all regions to be enabled from " + - Bytes.toString(tableName)); + LOG.debug("Sleeping= " + sleep + "ms, waiting for all regions to be " + + "enabled in " + Bytes.toString(tableName)); } try { - Thread.sleep(getPauseTime(tries)); + Thread.sleep(sleep); } catch (InterruptedException e) { // continue } @@ -348,8 +349,8 @@ public class HBaseAdmin { Bytes.toString(tableName)); } } - if (!isTableEnabled(tableName)) - throw new IOException("unable to enable table " + + if (!enabled) + throw new IOException("Unable to enable table " + Bytes.toString(tableName)); LOG.info("Enabled table " + Bytes.toString(tableName)); } @@ -385,9 +386,10 @@ public class HBaseAdmin { } // Wait until all regions are disabled - for (int tries = 0; - (tries < numRetries) && (!isTableDisabled(tableName)); - tries++) { + boolean disabled = false; + for (int tries = 0; tries < this.numRetries; tries++) { + disabled = isTableDisabled(tableName); + if (disabled) break; if (LOG.isDebugEnabled()) { LOG.debug("Sleep. Waiting for all regions to be disabled from " + Bytes.toString(tableName)); @@ -402,7 +404,7 @@ public class HBaseAdmin { Bytes.toString(tableName)); } } - if (!isTableDisabled(tableName)) { + if (!disabled) { throw new RegionException("Retries exhausted, it took too long to wait"+ " for the table " + Bytes.toString(tableName) + " to be disabled."); } diff --git a/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java index a5bfffd4be3..1e0e06489dd 100644 --- a/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ b/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -450,8 +450,8 @@ public class HConnectionManager implements HConstants { * Returns true if all regions are offline * Returns false in any other case */ - private boolean testTableOnlineState(byte[] tableName, - boolean online) throws IOException { + private boolean testTableOnlineState(byte[] tableName, boolean online) + throws IOException { if (!tableExists(tableName)) { throw new TableNotFoundException(Bytes.toString(tableName)); } @@ -459,7 +459,6 @@ public class HConnectionManager implements HConstants { // The root region is always enabled return true; } - int rowsScanned = 0; int rowsOffline = 0; byte[] startKey = @@ -468,6 +467,8 @@ public class HConnectionManager implements HConstants { HRegionInfo currentRegion = null; Scan scan = new Scan(startKey); scan.addColumn(CATALOG_FAMILY, REGIONINFO_QUALIFIER); + int rows = this.conf.getInt("hbase.meta.scanner.caching", 100); + scan.setCaching(rows); ScannerCallable s = new ScannerCallable(this, (Bytes.equals(tableName, HConstants.META_TABLE_NAME) ? HConstants.ROOT_TABLE_NAME : HConstants.META_TABLE_NAME), scan); @@ -482,10 +483,14 @@ public class HConnectionManager implements HConstants { currentRegion = s.getHRegionInfo(); Result r = null; Result [] rrs = null; - while ((rrs = getRegionServerWithRetries(s)) != null) { + do { + rrs = getRegionServerWithRetries(s); + if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) { + break; //exit completely + } r = rrs[0]; - byte [] value = r.getValue(HConstants.CATALOG_FAMILY, - HConstants.REGIONINFO_QUALIFIER); + byte [] value = r.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); if (value != null) { HRegionInfo info = Writables.getHRegionInfoOrNull(value); if (info != null) { @@ -495,20 +500,21 @@ public class HConnectionManager implements HConstants { } } } - } + } while(true); endKey = currentRegion.getEndKey(); - } while (!(endKey == null || + } while (!(endKey == null || Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY))); - } - finally { + } finally { s.setClose(); + // Doing below will call 'next' again and this will close the scanner + // Without it we leave scanners open. + getRegionServerWithRetries(s); } - boolean onlineOffline = - online ? rowsOffline == 0 : rowsOffline == rowsScanned; - return rowsScanned > 0 && onlineOffline; - + LOG.debug("Rowscanned=" + rowsScanned + ", rowsOffline=" + rowsOffline); + boolean onOffLine = online? rowsOffline == 0: rowsOffline == rowsScanned; + return rowsScanned > 0 && onOffLine; } - + private static class HTableDescriptorFinder implements MetaScanner.MetaScannerVisitor { byte[] tableName; diff --git a/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java index 1f399921a05..c8833cf64a9 100644 --- a/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java +++ b/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -94,20 +94,20 @@ public class ScannerCallable extends ServerCallable { } private void close() { - if (this.scannerId == -1L) { - return; - } - try { - this.server.close(this.scannerId); - } catch (IOException e) { - Log.warn("Ignore, probably already closed", e); - } - this.scannerId = -1L; + if (this.scannerId == -1L) { + return; + } + try { + this.server.close(this.scannerId); + } catch (IOException e) { + Log.warn("Ignore, probably already closed", e); + } + this.scannerId = -1L; } protected long openScanner() throws IOException { - return server.openScanner( - this.location.getRegionInfo().getRegionName(), scan); + return this.server.openScanner(this.location.getRegionInfo().getRegionName(), + this.scan); } protected Scan getScan() { @@ -118,7 +118,7 @@ public class ScannerCallable extends ServerCallable { * Call this when the next invocation of call should close the scanner */ public void setClose() { - closed = true; + this.closed = true; } /** @@ -146,4 +146,4 @@ public class ScannerCallable extends ServerCallable { public void setCaching(int caching) { this.caching = caching; } -} \ No newline at end of file +} diff --git a/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java b/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java index bf147942146..a211cfa17fa 100644 --- a/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java +++ b/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java @@ -20,9 +20,9 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,14 +32,15 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.ipc.HRegionInterface; import org.apache.hadoop.hbase.util.Writables; -/** Instantiated to enable or disable a table */ +/** + * Instantiated to enable or disable a table + */ class ChangeTableState extends TableOperation { private final Log LOG = LogFactory.getLog(this.getClass()); private boolean online; - + // Do in order. protected final Map> servedRegions = - new HashMap>(); - + new TreeMap>(); protected long lockid; ChangeTableState(final HMaster master, final byte [] tableName, @@ -51,14 +52,13 @@ class ChangeTableState extends TableOperation { @Override protected void processScanItem(String serverName, HRegionInfo info) { - if (isBeingServed(serverName)) { - HashSet regions = servedRegions.get(serverName); + HashSet regions = this.servedRegions.get(serverName); if (regions == null) { regions = new HashSet(); } regions.add(info); - servedRegions.put(serverName, regions); + this.servedRegions.put(serverName, regions); } } @@ -67,13 +67,13 @@ class ChangeTableState extends TableOperation { throws IOException { // Process regions not being served if (LOG.isDebugEnabled()) { - LOG.debug("processing unserved regions"); + LOG.debug("Processing unserved regions"); } - for (HRegionInfo i: unservedRegions) { + for (HRegionInfo i: this.unservedRegions) { if (i.isOffline() && i.isSplit()) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping region " + i.toString() + - " because it is offline because it has been split"); + " because it is offline and split"); } continue; } @@ -81,36 +81,36 @@ class ChangeTableState extends TableOperation { // Update meta table Put put = updateRegionInfo(i); server.put(m.getRegionName(), put); - Delete delete = new Delete(i.getRegionName()); delete.deleteColumns(CATALOG_FAMILY, SERVER_QUALIFIER); delete.deleteColumns(CATALOG_FAMILY, STARTCODE_QUALIFIER); server.delete(m.getRegionName(), delete); if (LOG.isDebugEnabled()) { - LOG.debug("Updated columns in row: " + i.getRegionNameAsString()); + LOG.debug("Removed server and startcode from row and set online=" + + this.online + ": " + i.getRegionNameAsString()); } - synchronized (master.regionManager) { - if (online) { + if (this.online) { // Bring offline regions on-line - if (!master.regionManager.regionIsOpening(i.getRegionNameAsString())) { - master.regionManager.setUnassigned(i, false); + if (!this.master.regionManager.regionIsOpening(i.getRegionNameAsString())) { + this.master.regionManager.setUnassigned(i, false); } } else { // Prevent region from getting assigned. - master.regionManager.removeRegion(i); + this.master.regionManager.removeRegion(i); } } } // Process regions currently being served if (LOG.isDebugEnabled()) { - LOG.debug("processing regions currently being served"); + LOG.debug("Processing regions currently being served"); } - synchronized (master.regionManager) { - for (Map.Entry> e: servedRegions.entrySet()) { + synchronized (this.master.regionManager) { + for (Map.Entry> e: + this.servedRegions.entrySet()) { String serverName = e.getKey(); - if (online) { + if (this.online) { LOG.debug("Already online"); continue; // Already being served } @@ -118,14 +118,15 @@ class ChangeTableState extends TableOperation { // Cause regions being served to be taken off-line and disabled for (HRegionInfo i: e.getValue()) { if (LOG.isDebugEnabled()) { - LOG.debug("adding region " + i.getRegionNameAsString() + " to kill list"); + LOG.debug("Adding region " + i.getRegionNameAsString() + + " to setClosing list"); } // this marks the regions to be closed - master.regionManager.setClosing(serverName, i, true); + this.master.regionManager.setClosing(serverName, i, true); } } } - servedRegions.clear(); + this.servedRegions.clear(); } protected Put updateRegionInfo(final HRegionInfo i) diff --git a/src/java/org/apache/hadoop/hbase/master/HMaster.java b/src/java/org/apache/hadoop/hbase/master/HMaster.java index 589071e737a..4e7117530c0 100644 --- a/src/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/java/org/apache/hadoop/hbase/master/HMaster.java @@ -844,8 +844,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface, Scan scan = new Scan(firstRowInTable); scan.addColumn(CATALOG_FAMILY, REGIONINFO_QUALIFIER); scan.addColumn(CATALOG_FAMILY, SERVER_QUALIFIER); - long scannerid = - srvr.openScanner(metaRegionName, scan); + long scannerid = srvr.openScanner(metaRegionName, scan); try { while (true) { Result data = srvr.next(scannerid); @@ -882,8 +881,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface, Scan scan = new Scan(firstRowInTable); scan.addColumn(CATALOG_FAMILY, REGIONINFO_QUALIFIER); scan.addColumn(CATALOG_FAMILY, SERVER_QUALIFIER); - long scannerid = - srvr.openScanner(metaRegionName, scan); + long scannerid = srvr.openScanner(metaRegionName, scan); try { while (true) { Result data = srvr.next(scannerid); diff --git a/src/java/org/apache/hadoop/hbase/master/RegionManager.java b/src/java/org/apache/hadoop/hbase/master/RegionManager.java index 790a6e160e7..082e21b6e37 100644 --- a/src/java/org/apache/hadoop/hbase/master/RegionManager.java +++ b/src/java/org/apache/hadoop/hbase/master/RegionManager.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.HServerInfo; import org.apache.hadoop.hbase.HServerLoad; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.regionserver.HLog; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -511,25 +512,30 @@ class RegionManager implements HConstants { LOG.info("Skipped " + skipped + " region(s) that are in transition states"); } + /* + * PathFilter that accepts hbase tables only. + */ static class TableDirFilter implements PathFilter { - public boolean accept(Path path) { // skip the region servers' log dirs && version file - // HBASE-1112 want to sperate the log dirs from table's data dirs by a special character. + // HBASE-1112 want to separate the log dirs from table's data dirs by a + // special character. String pathname = path.getName(); - return !pathname.startsWith("log_") && !pathname.equals(VERSION_FILE_NAME); + return !pathname.equals(HLog.HREGION_LOGDIR_NAME) && + !pathname.equals(VERSION_FILE_NAME); } } - - static class RegionDirFilter implements PathFilter { + /* + * PathFilter that accepts all but compaction.dir names. + */ + static class RegionDirFilter implements PathFilter { public boolean accept(Path path) { return !path.getName().equals(HREGION_COMPACTIONDIR_NAME); } - } - + /** * @return the rough number of the regions on fs * Note: this method simply counts the regions on fs by accumulating all the dirs @@ -538,10 +544,8 @@ class RegionManager implements HConstants { */ public int countRegionsOnFS() throws IOException { int regions = 0; - - FileStatus[] tableDirs = + FileStatus[] tableDirs = master.fs.listStatus(master.rootdir, new TableDirFilter()); - FileStatus[] regionDirs; RegionDirFilter rdf = new RegionDirFilter(); for(FileStatus tabledir : tableDirs) { @@ -550,7 +554,6 @@ class RegionManager implements HConstants { regions += regionDirs.length; } } - return regions; } @@ -1650,4 +1653,4 @@ class RegionManager implements HConstants { return Bytes.compareTo(getRegionName(), o.getRegionName()); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java b/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java index a98722e5734..e8846f2cb5a 100644 --- a/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java +++ b/src/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java @@ -56,12 +56,13 @@ abstract class RetryableMetaOperation implements Callable { protected T doWithRetries() throws IOException, RuntimeException { List exceptions = new ArrayList(); - for(int tries = 0; tries < master.numRetries; tries++) { - if (master.closed.get()) { + for (int tries = 0; tries < this.master.numRetries; tries++) { + if (this.master.closed.get()) { return null; } try { - this.server = master.connection.getHRegionConnection(m.getServer()); + this.server = + this.master.connection.getHRegionConnection(m.getServer()); return this.call(); } catch (IOException e) { if (e instanceof TableNotFoundException || @@ -94,8 +95,8 @@ abstract class RetryableMetaOperation implements Callable { LOG.debug("Exception in RetryableMetaOperation: ", e); throw new RuntimeException(e); } - sleeper.sleep(); + this.sleeper.sleep(); } return null; } -} +} \ No newline at end of file diff --git a/src/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/java/org/apache/hadoop/hbase/master/ServerManager.java index 5cb3bc93240..d802bf4d800 100644 --- a/src/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/src/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -477,7 +477,7 @@ class ServerManager implements HConstants { // Should we tell it close regions because its overloaded? If its // currently opening regions, leave it alone till all are open. - if ((openingCount < this.nobalancingCount)) { + if (openingCount < this.nobalancingCount) { this.master.regionManager.assignRegions(serverInfo, mostLoadedRegions, returnMsgs); } diff --git a/src/java/org/apache/hadoop/hbase/master/TableOperation.java b/src/java/org/apache/hadoop/hbase/master/TableOperation.java index f882a8eed1f..f80fe8d1cd9 100644 --- a/src/java/org/apache/hadoop/hbase/master/TableOperation.java +++ b/src/java/org/apache/hadoop/hbase/master/TableOperation.java @@ -21,9 +21,9 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.TreeSet; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; @@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.util.Bytes; abstract class TableOperation implements HConstants { private final Set metaRegions; protected final byte [] tableName; - protected final Set unservedRegions = new HashSet(); + // Do regions in order. + protected final Set unservedRegions = new TreeSet(); protected HMaster master; protected TableOperation(final HMaster master, final byte [] tableName) @@ -79,15 +80,17 @@ abstract class TableOperation implements HConstants { // Open a scanner on the meta region byte [] tableNameMetaStart = - Bytes.toBytes(Bytes.toString(tableName) + ",,"); + Bytes.toBytes(Bytes.toString(tableName) + ",,"); Scan scan = new Scan(tableNameMetaStart).addFamily(CATALOG_FAMILY); - long scannerId = server.openScanner(m.getRegionName(), scan); - + long scannerId = this.server.openScanner(m.getRegionName(), scan); + int rows = this.master.getConfiguration(). + getInt("hbase.meta.scanner.caching", 100); + scan.setCaching(rows); List emptyRows = new ArrayList(); try { while (true) { - Result values = server.next(scannerId); - if(values == null || values.isEmpty()) { + Result values = this.server.next(scannerId); + if (values == null || values.isEmpty()) { break; } HRegionInfo info = this.master.getHRegionInfo(values.getRow(), values); @@ -119,7 +122,7 @@ abstract class TableOperation implements HConstants { } finally { if (scannerId != -1L) { try { - server.close(scannerId); + this.server.close(scannerId); } catch (IOException e) { e = RemoteExceptionHandler.checkIOException(e); LOG.error("closing scanner", e); @@ -143,7 +146,6 @@ abstract class TableOperation implements HConstants { postProcessMeta(m, server); unservedRegions.clear(); - return Boolean.TRUE; } } diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index e954bad952a..34b195bb8e3 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1888,9 +1888,7 @@ public class HRegionServer implements HConstants, HRegionInterface, requestCount.incrementAndGet(); try { HRegion r = getRegion(regionName); - InternalScanner s = r.getScanner(scan); - long scannerId = addScanner(s); - return scannerId; + return addScanner(r.getScanner(scan)); } catch (Throwable t) { throw convertThrowableToIOE(cleanup(t, "Failed openScanner")); }