From 8d8dc87d0dd1e6039866543b4dd75fdb273ac2d7 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Thu, 13 Oct 2011 23:12:30 +0000 Subject: [PATCH] HBASE-3446 ProcessServerShutdown fails if META moves, orphaning lots of regions git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1183158 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 + .../org/apache/hadoop/hbase/HConstants.java | 4 - .../org/apache/hadoop/hbase/KeyValue.java | 1 + .../hadoop/hbase/LocalHBaseCluster.java | 16 +- .../hadoop/hbase/catalog/CatalogTracker.java | 261 ++++++-- .../hadoop/hbase/catalog/MetaEditor.java | 296 +++------ .../catalog/MetaMigrationRemovingHTD.java | 242 +++++++ .../hadoop/hbase/catalog/MetaReader.java | 625 ++++++++++-------- .../hadoop/hbase/client/HBaseAdmin.java | 8 +- .../hbase/client/HConnectionManager.java | 52 +- .../apache/hadoop/hbase/client/HTable.java | 11 +- .../hadoop/hbase/client/MetaScanner.java | 2 +- .../apache/hadoop/hbase/client/Result.java | 6 +- .../client/RetriesExhaustedException.java | 72 +- .../hadoop/hbase/client/ScannerCallable.java | 4 +- .../hadoop/hbase/client/ServerCallable.java | 42 +- .../hadoop/hbase/ipc/HMasterInterface.java | 2 +- .../hbase/master/AssignmentManager.java | 14 +- .../apache/hadoop/hbase/master/HMaster.java | 76 +-- .../master/handler/EnableTableHandler.java | 4 +- .../master/handler/ServerShutdownHandler.java | 34 +- .../hadoop/hbase/regionserver/HRegion.java | 4 +- .../hbase/regionserver/HRegionServer.java | 3 + .../regionserver/RegionServerServices.java | 2 +- .../hadoop/hbase/thrift/ThriftServer.java | 20 +- .../apache/hadoop/hbase/util/HBaseFsck.java | 3 +- .../hbase/zookeeper/MetaNodeTracker.java | 25 +- src/main/ruby/hbase/admin.rb | 9 +- .../hadoop/hbase/TestRegionRebalancing.java | 7 +- .../hbase/catalog/TestCatalogTracker.java | 193 ++++-- .../hbase/catalog/TestMetaReaderEditor.java | 158 ++++- .../TestMetaReaderEditorNoCluster.java | 177 +++++ .../client/HConnectionTestingUtility.java | 85 +++ .../apache/hadoop/hbase/client/TestHCM.java | 14 +- .../hbase/client/TestMetaMigration.java | 21 +- .../hbase/master/TestCatalogJanitor.java | 99 +-- .../master/TestDistributedLogSplitting.java | 6 +- .../hadoop/hbase/master/TestMaster.java | 26 +- .../hbase/master/TestMasterFailover.java | 20 +- .../hadoop/hbase/util/TestMergeTable.java | 2 + src/test/ruby/hbase/admin_test.rb | 23 +- src/test/ruby/shell/shell_test.rb | 2 +- 42 files changed, 1759 insertions(+), 914 deletions(-) create mode 100644 src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java create mode 100644 src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java create mode 100644 src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java diff --git a/CHANGES.txt b/CHANGES.txt index dbb75120450..36bedafd87d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -361,6 +361,8 @@ Release 0.92.0 - Unreleased names, need to use a more consistent block naming scheme (jgray) HBASE-4551 Fix pom and some test cases to compile and run against Hadoop 0.23 (todd) + HBASE-3446 ProcessServerShutdown fails if META moves, orphaning lots of + regions TESTS HBASE-4450 test for number of blocks read: to serve as baseline for expected diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java index 3af1bf03038..144f5269ef7 100644 --- a/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -185,10 +185,6 @@ public final class HConstants { /** The file name used to store HTD in HDFS */ public static final String TABLEINFO_NAME = ".tableinfo"; - /** The metaupdated column qualifier */ - public static final byte [] META_MIGRATION_QUALIFIER = Bytes.toBytes("metamigrated"); - - /** Default maximum file size */ public static final long DEFAULT_MAX_FILE_SIZE = 256 * 1024 * 1024; diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java index 0aad34b2db8..e709ec953af 100644 --- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -1043,6 +1043,7 @@ public class KeyValue implements Writable, HeapSize { * @return True if this KV is a {@link KeyValue.Type#Delete} type. */ public boolean isDeleteType() { + // TODO: Fix this method name vis-a-vis isDelete! return getType() == Type.Delete.getCode(); } diff --git a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java index 54b493913cb..d25b20cb83d 100644 --- a/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java +++ b/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.util.Threads; import java.util.concurrent.CopyOnWriteArrayList; import org.apache.hadoop.hbase.master.HMaster; @@ -238,6 +239,7 @@ public class LocalHBaseCluster { List list = getRegionServers(); for (JVMClusterUtil.RegionServerThread rst: list) { if (rst.isAlive()) liveServers.add(rst); + else LOG.info("Not alive " + rst.getName()); } return liveServers; } @@ -386,12 +388,12 @@ public class LocalHBaseCluster { */ public void join() { if (this.regionThreads != null) { - for(Thread t: this.regionThreads) { - if (t.isAlive()) { - try { - t.join(); + for(Thread t: this.regionThreads) { + if (t.isAlive()) { + try { + Threads.threadDumpingIsAlive(t); } catch (InterruptedException e) { - // continue + LOG.debug("Interrupted", e); } } } @@ -400,9 +402,9 @@ public class LocalHBaseCluster { for (Thread t : this.masterThreads) { if (t.isAlive()) { try { - t.join(); + Threads.threadDumpingIsAlive(t); } catch (InterruptedException e) { - // continue + LOG.debug("Interrupted", e); } } } diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java index 3570e6a0423..119105e73d3 100644 --- a/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java +++ b/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java @@ -1,6 +1,4 @@ /** - * Copyright 2010 The Apache Software Foundation - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -33,7 +31,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; -import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; @@ -60,30 +57,76 @@ import org.apache.hadoop.ipc.RemoteException; * interrupt waits and close up shop. */ public class CatalogTracker { + // TODO: This class needs a rethink. The original intent was that it would be + // the one-stop-shop for root and meta locations and that it would get this + // info from reading and watching zk state. The class was to be used by + // servers when they needed to know of root and meta movement but also by + // client-side (inside in HTable) so rather than figure root and meta + // locations on fault, the client would instead get notifications out of zk. + // + // But this original intent is frustrated by the fact that this class has to + // read an hbase table, the -ROOT- table, to figure out the .META. region + // location which means we depend on an HConnection. HConnection will do + // retrying but also, it has its own mechanism for finding root and meta + // locations (and for 'verifying'; it tries the location and if it fails, does + // new lookup, etc.). So, at least for now, HConnection (or HTable) can't + // have a CT since CT needs a HConnection (Even then, do want HT to have a CT? + // For HT keep up a session with ZK? Rather, shouldn't we do like asynchbase + // where we'd open a connection to zk, read what we need then let the + // connection go?). The 'fix' is make it so both root and meta addresses + // are wholey up in zk -- not in zk (root) -- and in an hbase table (meta). + // + // But even then, this class does 'verification' of the location and it does + // this by making a call over an HConnection (which will do its own root + // and meta lookups). Isn't this verification 'useless' since when we + // return, whatever is dependent on the result of this call then needs to + // use HConnection; what we have verified may change in meantime (HConnection + // uses the CT primitives, the root and meta trackers finding root locations). + // + // When meta is moved to zk, this class may make more sense. In the + // meantime, it does not cohere. It should just watch meta and root and not + // NOT do verification -- let that be out in HConnection since its going to + // be done there ultimately anyways. + // + // This class has spread throughout the codebase. It needs to be reigned in. + // This class should be used server-side only, even if we move meta location + // up into zk. Currently its used over in the client package. Its used in + // MetaReader and MetaEditor classes usually just to get the Configuration + // its using (It does this indirectly by asking its HConnection for its + // Configuration and even then this is just used to get an HConnection out on + // the other end). I made https://issues.apache.org/jira/browse/HBASE-4495 for + // doing CT fixup. St.Ack 09/30/2011. + // private static final Log LOG = LogFactory.getLog(CatalogTracker.class); - private final Configuration conf; private final HConnection connection; private final ZooKeeperWatcher zookeeper; private final RootRegionTracker rootRegionTracker; private final MetaNodeTracker metaNodeTracker; private final AtomicBoolean metaAvailable = new AtomicBoolean(false); - /** + private boolean instantiatedzkw = false; + + /* * Do not clear this address once set. Its needed when we do * server shutdown processing -- we need to know who had .META. last. If you * want to know if the address is good, rely on {@link #metaAvailable} value. */ private ServerName metaLocation; + + /* + * Timeout waiting on root or meta to be set. + */ private final int defaultTimeout; + private boolean stopped = false; - public static final byte [] ROOT_REGION = + static final byte [] ROOT_REGION_NAME = HRegionInfo.ROOT_REGIONINFO.getRegionName(); - public static final byte [] META_REGION = + static final byte [] META_REGION_NAME = HRegionInfo.FIRST_META_REGIONINFO.getRegionName(); /** - * Constructs a catalog tracker. Find current state of catalog tables and - * begin active tracking by executing {@link #start()} post construction. Does + * Constructs a catalog tracker. Find current state of catalog tables. + * Begin active tracking by executing {@link #start()} post construction. Does * not timeout. * * @param conf @@ -97,29 +140,36 @@ public class CatalogTracker { } /** - * Constructs the catalog tracker. Find current state of catalog tables and - * begin active tracking by executing {@link #start()} post construction. + * Constructs the catalog tracker. Find current state of catalog tables. + * Begin active tracking by executing {@link #start()} post construction. * Does not timeout. - * @param zk + * @param zk If zk is null, we'll create an instance (and shut it down + * when {@link #stop()} is called) else we'll use what is passed. * @param connection server connection - * @param abortable if fatal exception + * @param abortable If fatal exception we'll call abort on this. May be null. + * If it is we'll use the Connection associated with the passed + * {@link Configuration} as our Abortable. * @throws IOException */ public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf, final Abortable abortable) throws IOException { - this(zk, conf, abortable, 0); + this(zk, conf, abortable, + conf.getInt("hbase.catalogtracker.default.timeout", 1000)); } /** - * Constructs the catalog tracker. Find current state of catalog tables and - * begin active tracking by executing {@link #start()} post construction. - * @param zk - * @param connection server connection - * @param abortable if fatal exception + * Constructs the catalog tracker. Find current state of catalog tables. + * Begin active tracking by executing {@link #start()} post construction. + * @param zk If zk is null, we'll create an instance (and shut it down + * when {@link #stop()} is called) else we'll use what is passed. + * @param conf + * @param abortable If fatal exception we'll call abort on this. May be null. + * If it is we'll use the Connection associated with the passed + * {@link Configuration} as our Abortable. * @param defaultTimeout Timeout to use. Pass zero for no timeout * ({@link Object#wait(long)} when passed a 0 waits for ever). - * @throws IOException + * @throws IOException */ public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf, Abortable abortable, final int defaultTimeout) @@ -130,14 +180,29 @@ public class CatalogTracker { CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf, HConnection connection, Abortable abortable, final int defaultTimeout) throws IOException { - this.conf = conf; this.connection = connection; - this.zookeeper = (zk == null) ? this.connection.getZooKeeperWatcher() : zk; if (abortable == null) { + // A connection is abortable. abortable = this.connection; } + if (zk == null) { + // Create our own. Set flag so we tear it down on stop. + this.zookeeper = + new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(), + abortable); + instantiatedzkw = true; + } else { + this.zookeeper = zk; + } this.rootRegionTracker = new RootRegionTracker(zookeeper, abortable); - this.metaNodeTracker = new MetaNodeTracker(zookeeper, this, abortable); + final CatalogTracker ct = this; + // Override nodeDeleted so we get notified when meta node deleted + this.metaNodeTracker = new MetaNodeTracker(zookeeper, abortable) { + public void nodeDeleted(String path) { + if (!path.equals(node)) return; + ct.resetMetaLocation(); + } + }; this.defaultTimeout = defaultTimeout; } @@ -149,9 +214,9 @@ public class CatalogTracker { * @throws InterruptedException */ public void start() throws IOException, InterruptedException { + LOG.debug("Starting catalog tracker " + this); this.rootRegionTracker.start(); this.metaNodeTracker.start(); - LOG.debug("Starting catalog tracker " + this); } /** @@ -173,6 +238,9 @@ public class CatalogTracker { // IOException}, in reality, the implementation would never do that. LOG.error("Attempt to close catalog tracker's connection failed.", e); } + if (this.instantiatedzkw) { + this.zookeeper.close(); + } // Call this and it will interrupt any ongoing waits on meta. synchronized (this.metaAvailable) { this.metaAvailable.notifyAll(); @@ -183,7 +251,8 @@ public class CatalogTracker { /** * Gets the current location for -ROOT- or null if location is * not currently available. - * @return server name + * @return {@link ServerName} for server hosting -ROOT- or null + * if none available * @throws InterruptedException */ public ServerName getRootLocation() throws InterruptedException { @@ -191,8 +260,8 @@ public class CatalogTracker { } /** - * @return Location of server hosting meta region formatted as per - * {@link ServerName}, or null if none available + * @return {@link ServerName} for server hosting .META. or null + * if none available */ public ServerName getMetaLocation() { return this.metaLocation; @@ -213,7 +282,8 @@ public class CatalogTracker { * for up to the specified timeout if not immediately available. Returns null * if the timeout elapses before root is available. * @param timeout maximum time to wait for root availability, in milliseconds - * @return Location of server hosting root region or null if none available + * @return {@link ServerName} for server hosting -ROOT- or null + * if none available * @throws InterruptedException if interrupted while waiting * @throws NotAllMetaRegionsOnlineException if root not available before * timeout @@ -230,13 +300,31 @@ public class CatalogTracker { /** * Gets a connection to the server hosting root, as reported by ZooKeeper, * waiting up to the specified timeout for availability. + * @param timeout How long to wait on root location + * @see #waitForRoot(long) for additional information + * @return connection to server hosting root + * @throws InterruptedException + * @throws NotAllMetaRegionsOnlineException if timed out waiting + * @throws IOException + * @deprecated Use {@link #getRootServerConnection(long)} + */ + public HRegionInterface waitForRootServerConnection(long timeout) + throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { + return getRootServerConnection(timeout); + } + + /** + * Gets a connection to the server hosting root, as reported by ZooKeeper, + * waiting up to the specified timeout for availability. + *

WARNING: Does not retry. Use an {@link HTable} instead. + * @param timeout How long to wait on root location * @see #waitForRoot(long) for additional information * @return connection to server hosting root * @throws InterruptedException * @throws NotAllMetaRegionsOnlineException if timed out waiting * @throws IOException */ - public HRegionInterface waitForRootServerConnection(long timeout) + HRegionInterface getRootServerConnection(long timeout) throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { return getCachedConnection(waitForRoot(timeout)); } @@ -248,29 +336,17 @@ public class CatalogTracker { * @return connection to server hosting root * @throws NotAllMetaRegionsOnlineException if timed out waiting * @throws IOException + * @deprecated Use {@link #getRootServerConnection(long)} */ public HRegionInterface waitForRootServerConnectionDefault() throws NotAllMetaRegionsOnlineException, IOException { try { - return getCachedConnection(waitForRoot(defaultTimeout)); + return getRootServerConnection(this.defaultTimeout); } catch (InterruptedException e) { throw new NotAllMetaRegionsOnlineException("Interrupted"); } } - /** - * Gets a connection to the server hosting root, as reported by ZooKeeper, - * if available. Returns null if no location is immediately available. - * @return connection to server hosting root, null if not available - * @throws IOException - * @throws InterruptedException - */ - private HRegionInterface getRootServerConnection() - throws IOException, InterruptedException { - ServerName sn = this.rootRegionTracker.getRootRegionLocation(); - return getCachedConnection(sn); - } - /** * Gets a connection to the server currently hosting .META. or * null if location is not currently available. @@ -292,24 +368,28 @@ public class CatalogTracker { synchronized (metaAvailable) { if (metaAvailable.get()) { HRegionInterface current = getCachedConnection(this.metaLocation); - if (verifyRegionLocation(current, this.metaLocation, META_REGION)) { + // If we are to refresh, verify we have a good connection by making + // an invocation on it. + if (verifyRegionLocation(current, this.metaLocation, META_REGION_NAME)) { return current; } resetMetaLocation(); } - HRegionInterface rootConnection = getRootServerConnection(); - if (rootConnection == null) { - LOG.debug("-ROOT- server unavailable."); - return null; - } - ServerName newLocation = MetaReader.readMetaLocation(rootConnection); + // We got here because there is no meta available or because whats + // available is bad. + + // Now read the current .META. content from -ROOT-. Note: This goes via + // an HConnection. It has its own way of figuring root and meta locations + // which we have to wait on. + ServerName newLocation = + MetaReader.readRegionLocation(this, META_REGION_NAME); if (newLocation == null) { LOG.debug(".META. server unavailable."); return null; } HRegionInterface newConnection = getCachedConnection(newLocation); - if (verifyRegionLocation(newConnection, newLocation, META_REGION)) { + if (verifyRegionLocation(newConnection, newLocation, META_REGION_NAME)) { setMetaLocation(newLocation); return newConnection; } else { @@ -322,13 +402,19 @@ public class CatalogTracker { /** * Waits indefinitely for availability of .META.. Used during - * cluster startup. + * cluster startup. Does not verify meta, just that something has been + * set up in zk. + * @see #waitForMeta(long) * @throws InterruptedException if interrupted while waiting */ public void waitForMeta() throws InterruptedException { - synchronized (metaAvailable) { - while (!stopped && !metaAvailable.get()) { - metaAvailable.wait(); + while (!this.stopped) { + try { + if (waitForMeta(100) != null) break; + } catch (NotAllMetaRegionsOnlineException e) { + LOG.info("Retrying", e); + } catch (IOException e) { + LOG.info("Retrying", e); } } } @@ -340,7 +426,8 @@ public class CatalogTracker { * in that it will go ahead and verify the location gotten from ZooKeeper and * -ROOT- region by trying to use returned connection. * @param timeout maximum time to wait for meta availability, in milliseconds - * @return location of meta + * @return {@link ServerName} for server hosting .META. or null + * if none available * @throws InterruptedException if interrupted while waiting * @throws IOException unexpected exception connecting to meta server * @throws NotAllMetaRegionsOnlineException if meta not available before @@ -359,8 +446,7 @@ public class CatalogTracker { metaAvailable.wait(waitTime); } if (getMetaServerConnection() == null) { - throw new NotAllMetaRegionsOnlineException( - "Timed out (" + timeout + "ms)"); + throw new NotAllMetaRegionsOnlineException("Timed out (" + timeout + "ms)"); } return metaLocation; } @@ -374,6 +460,7 @@ public class CatalogTracker { * @throws InterruptedException * @throws NotAllMetaRegionsOnlineException if timed out waiting * @throws IOException + * @deprecated Does not retry; use an HTable instance instead. */ public HRegionInterface waitForMetaServerConnection(long timeout) throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { @@ -383,10 +470,12 @@ public class CatalogTracker { /** * Gets a connection to the server hosting meta, as reported by ZooKeeper, * waiting up to the specified timeout for availability. + * Used in tests. * @see #waitForMeta(long) for additional information * @return connection to server hosting meta * @throws NotAllMetaRegionsOnlineException if timed out or interrupted * @throws IOException + * @deprecated Does not retry; use an HTable instance instead. */ public HRegionInterface waitForMetaServerConnectionDefault() throws NotAllMetaRegionsOnlineException, IOException { @@ -397,12 +486,22 @@ public class CatalogTracker { } } - private void resetMetaLocation() { - LOG.debug("Current cached META location: " + metaLocation + - " is not valid, resetting"); - this.metaAvailable.set(false); + /** + * Called when we figure current meta is off (called from zk callback). + */ + public void resetMetaLocation() { + LOG.debug("Current cached META location, " + metaLocation + + ", is not valid, resetting"); + synchronized(this.metaAvailable) { + this.metaAvailable.set(false); + this.metaAvailable.notifyAll(); + } } + /** + * Caller must be synchronized on this.metaAvailable + * @param metaLocation + */ private void setMetaLocation(final ServerName metaLocation) { LOG.debug("set new cached META location: " + metaLocation); metaAvailable.set(true); @@ -411,6 +510,13 @@ public class CatalogTracker { this.metaAvailable.notifyAll(); } + /** + * @param sn ServerName to get a connection against. + * @return The HRegionInterface we got when we connected to sn + * May have come from cache, may not be good, may have been setup by this + * invocation, or may be null. + * @throws IOException + */ private HRegionInterface getCachedConnection(ServerName sn) throws IOException { if (sn == null) { @@ -446,17 +552,32 @@ public class CatalogTracker { return protocol; } - private boolean verifyRegionLocation(HRegionInterface metaServer, - final ServerName address, - byte [] regionName) + /** + * Verify we can connect to hostingServer and that its carrying + * regionName. + * @param hostingServer Interface to the server hosting regionName + * @param serverName The servername that goes with the metaServer + * Interface. Used logging. + * @param regionName The regionname we are interested in. + * @return True if we were able to verify the region located at other side of + * the Interface. + * @throws IOException + */ + // TODO: We should be able to get the ServerName from the HRegionInterface + // rather than have to pass it in. Its made awkward by the fact that the + // HRI is likely a proxy against remote server so the getServerName needs + // to be fixed to go to a local method or to a cache before we can do this. + private boolean verifyRegionLocation(HRegionInterface hostingServer, + final ServerName address, final byte [] regionName) throws IOException { - if (metaServer == null) { - LOG.info("Passed metaserver is null"); + if (hostingServer == null) { + LOG.info("Passed hostingServer is null"); return false; } Throwable t = null; try { - return metaServer.getRegionInfo(regionName) != null; + // Try and get regioninfo from the hosting server. + return hostingServer.getRegionInfo(regionName) != null; } catch (ConnectException e) { t = e; } catch (RemoteException e) { @@ -498,8 +619,7 @@ public class CatalogTracker { } return (connection == null)? false: verifyRegionLocation(connection, - this.rootRegionTracker.getRootRegionLocation(), - HRegionInfo.ROOT_REGIONINFO.getRegionName()); + this.rootRegionTracker.getRootRegionLocation(), ROOT_REGION_NAME); } /** @@ -523,6 +643,7 @@ public class CatalogTracker { return connection != null; } + // Used by tests. MetaNodeTracker getMetaNodeTracker() { return this.metaNodeTracker; } diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java index ac603118de6..6387df977a0 100644 --- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java +++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java @@ -1,6 +1,4 @@ /** - * Copyright 2010 The Apache Software Foundation - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,40 +17,121 @@ */ package org.apache.hadoop.hbase.catalog; +import java.io.IOException; +import java.net.ConnectException; import java.util.ArrayList; import java.util.List; -import java.io.IOException; -import java.net.ConnectException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.ipc.HRegionInterface; -import org.apache.hadoop.hbase.migration.HRegionInfo090x; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Writables; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.catalog.MetaReader.Visitor; /** * Writes region and assignment information to .META.. - *

- * Uses the {@link CatalogTracker} to obtain locations and connections to - * catalogs. */ public class MetaEditor { + // TODO: Strip CatalogTracker from this class. Its all over and in the end + // its only used to get its Configuration so we can get associated + // Connection. private static final Log LOG = LogFactory.getLog(MetaEditor.class); - private static Put makePutFromRegionInfo(HRegionInfo regionInfo) throws IOException { + private static Put makePutFromRegionInfo(HRegionInfo regionInfo) + throws IOException { Put put = new Put(regionInfo.getRegionName()); put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(regionInfo)); return put; } + + /** + * Put the passed p to the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param p Put to add to .META. + * @throws IOException + */ + static void putToMetaTable(final CatalogTracker ct, final Put p) + throws IOException { + put(MetaReader.getMetaHTable(ct), p); + } + + /** + * Put the passed p to the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param p Put to add to .META. + * @throws IOException + */ + static void putToRootTable(final CatalogTracker ct, final Put p) + throws IOException { + put(MetaReader.getRootHTable(ct), p); + } + + /** + * Put the passed p to a catalog table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param regionName Name of the catalog table to put too. + * @param p Put to add + * @throws IOException + */ + static void putToCatalogTable(final CatalogTracker ct, + final byte [] regionName, final Put p) + throws IOException { + HTable t = MetaReader.getCatalogHTable(ct, regionName); + put(t, p); + } + + /** + * @param t Table to use (will be closed when done). + * @param p + * @throws IOException + */ + private static void put(final HTable t, final Put p) throws IOException { + try { + t.put(p); + } finally { + t.close(); + } + } + + /** + * Put the passed ps to the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param ps Put to add to .META. + * @throws IOException + */ + static void putsToMetaTable(final CatalogTracker ct, final List ps) + throws IOException { + HTable t = MetaReader.getMetaHTable(ct); + try { + t.put(ps); + } finally { + t.close(); + } + } + + /** + * Delete the passed d from the .META. table. + * @param ct CatalogTracker on whose back we will ride the edit. + * @param d Delete to add to .META. + * @throws IOException + */ + static void deleteMetaTable(final CatalogTracker ct, final Delete d) + throws IOException { + HTable t = MetaReader.getMetaHTable(ct); + try { + t.delete(d); + } finally { + t.close(); + } + } /** * Adds a META row for the specified new region. @@ -62,8 +141,7 @@ public class MetaEditor { public static void addRegionToMeta(CatalogTracker catalogTracker, HRegionInfo regionInfo) throws IOException { - catalogTracker.waitForMetaServerConnectionDefault().put( - CatalogTracker.META_REGION, makePutFromRegionInfo(regionInfo)); + putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo)); LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to META"); } @@ -79,11 +157,9 @@ public class MetaEditor { List puts = new ArrayList(); for (HRegionInfo regionInfo : regionInfos) { puts.add(makePutFromRegionInfo(regionInfo)); - LOG.debug("Added region " + regionInfo.getRegionNameAsString() + " to META"); } - catalogTracker.waitForMetaServerConnectionDefault().put( - CatalogTracker.META_REGION, puts); - LOG.info("Added " + puts.size() + " regions to META"); + putsToMetaTable(catalogTracker, puts); + LOG.info("Added " + puts.size() + " regions in META"); } /** @@ -108,7 +184,7 @@ public class MetaEditor { Writables.getBytes(a)); put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER, Writables.getBytes(b)); - catalogTracker.waitForMetaServerConnectionDefault().put(CatalogTracker.META_REGION, put); + putToMetaTable(catalogTracker, put); LOG.info("Offlined parent region " + parent.getRegionNameAsString() + " in META"); } @@ -116,14 +192,11 @@ public class MetaEditor { public static void addDaughter(final CatalogTracker catalogTracker, final HRegionInfo regionInfo, final ServerName sn) throws NotAllMetaRegionsOnlineException, IOException { - HRegionInterface server = catalogTracker.waitForMetaServerConnectionDefault(); - byte [] catalogRegionName = CatalogTracker.META_REGION; Put put = new Put(regionInfo.getRegionName()); addRegionInfo(put, regionInfo); if (sn != null) addLocation(put, sn); - server.put(catalogRegionName, put); + putToMetaTable(catalogTracker, put); LOG.info("Added daughter " + regionInfo.getRegionNameAsString() + - " in region " + Bytes.toString(catalogRegionName) + (sn == null? ", serverName=null": ", serverName=" + sn.toString())); } @@ -145,9 +218,7 @@ public class MetaEditor { public static void updateMetaLocation(CatalogTracker catalogTracker, HRegionInfo regionInfo, ServerName sn) throws IOException, ConnectException { - HRegionInterface server = catalogTracker.waitForRootServerConnectionDefault(); - if (server == null) throw new IOException("No server for -ROOT-"); - updateLocation(server, CatalogTracker.ROOT_REGION, regionInfo, sn); + updateLocation(catalogTracker, regionInfo, sn); } /** @@ -165,8 +236,7 @@ public class MetaEditor { public static void updateRegionLocation(CatalogTracker catalogTracker, HRegionInfo regionInfo, ServerName sn) throws IOException { - updateLocation(catalogTracker.waitForMetaServerConnectionDefault(), - CatalogTracker.META_REGION, regionInfo, sn); + updateLocation(catalogTracker, regionInfo, sn); } /** @@ -175,22 +245,21 @@ public class MetaEditor { * Connects to the specified server which should be hosting the specified * catalog region name to perform the edit. * - * @param server connection to server hosting catalog region - * @param catalogRegionName name of catalog region being updated + * @param catalogTracker * @param regionInfo region to update location of * @param sn Server name * @throws IOException In particular could throw {@link java.net.ConnectException} * if the server is down on other end. */ - private static void updateLocation(HRegionInterface server, - byte [] catalogRegionName, HRegionInfo regionInfo, ServerName sn) + private static void updateLocation(final CatalogTracker catalogTracker, + HRegionInfo regionInfo, ServerName sn) throws IOException { + final byte [] regionName = regionInfo.getRegionName(); Put put = new Put(regionInfo.getRegionName()); addLocation(put, sn); - server.put(catalogRegionName, put); + putToCatalogTable(catalogTracker, regionName, put); LOG.info("Updated row " + regionInfo.getRegionNameAsString() + - " in region " + Bytes.toStringBinary(catalogRegionName) + " with " + - "serverName=" + sn.toString()); + " with server=" + sn); } /** @@ -203,8 +272,7 @@ public class MetaEditor { HRegionInfo regionInfo) throws IOException { Delete delete = new Delete(regionInfo.getRegionName()); - catalogTracker.waitForMetaServerConnectionDefault(). - delete(CatalogTracker.META_REGION, delete); + deleteMetaTable(catalogTracker, delete); LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from META"); } @@ -223,142 +291,12 @@ public class MetaEditor { throws NotAllMetaRegionsOnlineException, IOException { Delete delete = new Delete(parent.getRegionName()); delete.deleteColumns(HConstants.CATALOG_FAMILY, qualifier); - catalogTracker.waitForMetaServerConnectionDefault(). - delete(CatalogTracker.META_REGION, delete); + deleteMetaTable(catalogTracker, delete); LOG.info("Deleted daughter reference " + daughter.getRegionNameAsString() + ", qualifier=" + Bytes.toStringBinary(qualifier) + ", from parent " + parent.getRegionNameAsString()); } - /** - * Update the metamigrated flag in -ROOT-. - * @param catalogTracker - * @throws IOException - */ - public static void updateRootWithMetaMigrationStatus( - CatalogTracker catalogTracker) throws IOException { - updateRootWithMetaMigrationStatus(catalogTracker, true); - } - - /** - * Update the metamigrated flag in -ROOT-. - * @param catalogTracker - * @param metaUpdated - * @throws IOException - */ - public static void updateRootWithMetaMigrationStatus( - CatalogTracker catalogTracker, boolean metaUpdated) - throws IOException { - Put put = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName()); - addMetaUpdateStatus(put, metaUpdated); - catalogTracker.waitForRootServerConnectionDefault().put( - CatalogTracker.ROOT_REGION, put); - LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated); - } - - /** - * Update legacy META rows, removing HTD from HRI. - * @param masterServices - * @return - * @throws IOException - */ - public static List updateMetaWithNewRegionInfo( - final MasterServices masterServices) - throws IOException { - final List htds = new ArrayList(); - Visitor v = new Visitor() { - @Override - public boolean visit(Result r) throws IOException { - if (r == null || r.isEmpty()) return true; - HRegionInfo090x hrfm = getHRegionInfoForMigration(r); - if (hrfm == null) return true; - htds.add(hrfm.getTableDesc()); - masterServices.getMasterFileSystem() - .createTableDescriptor(hrfm.getTableDesc()); - updateHRI(masterServices.getCatalogTracker() - .waitForMetaServerConnectionDefault(), - hrfm, CatalogTracker.META_REGION); - return true; - } - }; - MetaReader.fullScan(masterServices.getCatalogTracker(), v); - updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker()); - return htds; - } - - /** - * Migrate root and meta to newer version. This updates the META and ROOT - * and removes the HTD from HRI. - * @param masterServices - * @throws IOException - */ - public static void migrateRootAndMeta(final MasterServices masterServices) - throws IOException { - updateRootWithNewRegionInfo(masterServices); - updateMetaWithNewRegionInfo(masterServices); - } - - /** - * Update the ROOT with new HRI. (HRI with no HTD) - * @param masterServices - * @return - * @throws IOException - */ - public static List updateRootWithNewRegionInfo( - final MasterServices masterServices) - throws IOException { - final List htds = new ArrayList(); - Visitor v = new Visitor() { - @Override - public boolean visit(Result r) throws IOException { - if (r == null || r.isEmpty()) return true; - HRegionInfo090x hrfm = getHRegionInfoForMigration(r); - if (hrfm == null) return true; - htds.add(hrfm.getTableDesc()); - masterServices.getMasterFileSystem().createTableDescriptor( - hrfm.getTableDesc()); - updateHRI(masterServices.getCatalogTracker() - .waitForRootServerConnectionDefault(), - hrfm, CatalogTracker.ROOT_REGION); - return true; - } - }; - MetaReader.fullScan( - masterServices.getCatalogTracker().waitForRootServerConnectionDefault(), - v, HRegionInfo.ROOT_REGIONINFO.getRegionName(), null); - return htds; - } - - private static void updateHRI(HRegionInterface hRegionInterface, - HRegionInfo090x hRegionInfo090x, byte[] regionName) - throws IOException { - HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x); - Put put = new Put(regionInfo.getRegionName()); - put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, - Writables.getBytes(regionInfo)); - hRegionInterface.put(regionName, put); - LOG.info("Updated region " + regionInfo + " to " + Bytes.toString(regionName)); - } - - public static HRegionInfo090x getHRegionInfoForMigration( - Result data) throws IOException { - HRegionInfo090x info = null; - byte [] bytes = - data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); - if (bytes == null) return null; - try { - info = Writables.getHRegionInfoForMigration(bytes); - } catch(IOException ioe) { - if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) { - return null; - } else { - throw ioe; - } - } - LOG.info("Current INFO from scan results = " + info); - return info; - } - public static HRegionInfo getHRegionInfo( Result data) throws IOException { byte [] bytes = @@ -369,20 +307,6 @@ public class MetaEditor { return info; } - private static Put addMetaUpdateStatus(final Put p) { - p.add(HConstants.CATALOG_FAMILY, HConstants.META_MIGRATION_QUALIFIER, - Bytes.toBytes("true")); - return p; - } - - - private static Put addMetaUpdateStatus(final Put p, final boolean metaUpdated) { - p.add(HConstants.CATALOG_FAMILY, HConstants.META_MIGRATION_QUALIFIER, - Bytes.toBytes(metaUpdated)); - return p; - } - - private static Put addRegionInfo(final Put p, final HRegionInfo hri) throws IOException { p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, @@ -397,4 +321,4 @@ public class MetaEditor { Bytes.toBytes(sn.getStartcode())); return p; } -} +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java new file mode 100644 index 00000000000..0c1d221a677 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.catalog.MetaReader.Visitor; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.migration.HRegionInfo090x; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; + +/** + * Tools to help with migration of meta tables so they no longer host + * instances of HTableDescriptor. + * @deprecated Used migration from 0.90 to 0.92 so will be going away in next + * release + */ +public class MetaMigrationRemovingHTD { + private static final Log LOG = LogFactory.getLog(MetaMigrationRemovingHTD.class); + + /** The metaupdated column qualifier */ + public static final byte [] META_MIGRATION_QUALIFIER = + Bytes.toBytes("metamigrated"); + + /** + * Update legacy META rows, removing HTD from HRI. + * @param masterServices + * @return + * @throws IOException + */ + public static List updateMetaWithNewRegionInfo( + final MasterServices masterServices) + throws IOException { + final List htds = new ArrayList(); + Visitor v = new Visitor() { + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r); + if (hrfm == null) return true; + htds.add(hrfm.getTableDesc()); + masterServices.getMasterFileSystem() + .createTableDescriptor(hrfm.getTableDesc()); + updateHRI(masterServices.getCatalogTracker(), false, hrfm); + return true; + } + }; + MetaReader.fullScan(masterServices.getCatalogTracker(), v); + MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker(), true); + return htds; + } + + /** + * Update the ROOT with new HRI. (HRI with no HTD) + * @param masterServices + * @return + * @throws IOException + */ + public static List updateRootWithNewRegionInfo( + final MasterServices masterServices) + throws IOException { + final List htds = new ArrayList(); + Visitor v = new Visitor() { + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r); + if (hrfm == null) return true; + htds.add(hrfm.getTableDesc()); + masterServices.getMasterFileSystem().createTableDescriptor( + hrfm.getTableDesc()); + updateHRI(masterServices.getCatalogTracker(), true, hrfm); + return true; + } + }; + MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true); + return htds; + } + + /** + * Migrate root and meta to newer version. This updates the META and ROOT + * and removes the HTD from HRI. + * @param masterServices + * @throws IOException + */ + public static void migrateRootAndMeta(final MasterServices masterServices) + throws IOException { + updateRootWithNewRegionInfo(masterServices); + updateMetaWithNewRegionInfo(masterServices); + } + + /** + * Update the metamigrated flag in -ROOT-. + * @param catalogTracker + * @param metaUpdated + * @throws IOException + */ + public static void updateRootWithMetaMigrationStatus( + CatalogTracker catalogTracker, boolean metaUpdated) + throws IOException { + Put p = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName()); + MetaMigrationRemovingHTD.addMetaUpdateStatus(p, metaUpdated); + MetaEditor.putToRootTable(catalogTracker, p); + LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated); + } + + static void updateHRI(final CatalogTracker ct, final boolean rootTable, + final HRegionInfo090x hRegionInfo090x) + throws IOException { + HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x); + Put p = new Put(regionInfo.getRegionName()); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(regionInfo)); + if (rootTable) { + MetaEditor.putToRootTable(ct, p); + } else { + MetaEditor.putToMetaTable(ct, p); + } + LOG.info("Updated region " + regionInfo + " to " + + (rootTable? "-ROOT-": ".META.")); + } + + /** + * @deprecated Going away in 0.94; used for migrating to 0.92 only. + */ + public static HRegionInfo090x getHRegionInfoForMigration( + Result data) throws IOException { + HRegionInfo090x info = null; + byte [] bytes = + data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + if (bytes == null) return null; + try { + info = Writables.getHRegionInfoForMigration(bytes); + } catch(IOException ioe) { + if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) { + return null; + } else { + throw ioe; + } + } + LOG.info("Current INFO from scan results = " + info); + return info; + } + + public static List fullScanMetaAndPrintHRIM( + CatalogTracker catalogTracker) + throws IOException { + final List regions = + new ArrayList(); + Visitor v = new Visitor() { + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + LOG.info("fullScanMetaAndPrint1.Current Meta Result: " + r); + HRegionInfo090x hrim = getHRegionInfoForMigration(r); + LOG.info("fullScanMetaAndPrint.HRIM Print= " + hrim); + regions.add(hrim); + return true; + } + }; + MetaReader.fullScan(catalogTracker, v); + return regions; + } + + static Put addMetaUpdateStatus(final Put p, final boolean metaUpdated) { + p.add(HConstants.CATALOG_FAMILY, + MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER, + Bytes.toBytes(metaUpdated)); + return p; + } + + /** + * @return True if the meta table has been migrated. + * @throws IOException + */ + // Public because used in tests + public static boolean isMetaHRIUpdated(final MasterServices services) + throws IOException { + boolean metaUpdated = false; + List results = + MetaReader.fullScanOfRoot(services.getCatalogTracker()); + if (results == null || results.isEmpty()) { + LOG.info("metaUpdated = NULL."); + return metaUpdated; + } + // Presume only the one result. + Result r = results.get(0); + byte [] metaMigrated = r.getValue(HConstants.CATALOG_FAMILY, + MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER); + if (metaMigrated != null && metaMigrated.length > 0) { + metaUpdated = Bytes.toBoolean(metaMigrated); + } + LOG.info("Meta updated status = " + metaUpdated); + return metaUpdated; + } + + /** + * @return True if migrated. + * @throws IOException + */ + public static boolean updateMetaWithNewHRI(final MasterServices services) + throws IOException { + if (isMetaHRIUpdated(services)) { + LOG.info("ROOT/Meta already up-to date with new HRI."); + return true; + } + LOG.info("Meta has HRI with HTDs. Updating meta now."); + try { + migrateRootAndMeta(services); + LOG.info("ROOT and Meta updated with new HRI."); + return true; + } catch (IOException e) { + throw new RuntimeException("Update ROOT/Meta with new HRI failed." + + "Master startup aborted."); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java index 2afe70cdc30..f7f4d606967 100644 --- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java +++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java @@ -1,6 +1,4 @@ /** - * Copyright 2010 The Apache Software Foundation - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -26,16 +24,20 @@ import java.util.Map; import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.ipc.HRegionInterface; -import org.apache.hadoop.hbase.migration.HRegionInfo090x; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Writables; @@ -43,14 +45,14 @@ import org.apache.hadoop.ipc.RemoteException; /** * Reads region and assignment information from .META.. - *

- * Uses the {@link CatalogTracker} to obtain locations and connections to - * catalogs. */ public class MetaReader { + // TODO: Strip CatalogTracker from this class. Its all over and in the end + // its only used to get its Configuration so we can get associated + // Connection. private static final Log LOG = LogFactory.getLog(MetaReader.class); - public static final byte [] META_REGION_PREFIX; + static final byte [] META_REGION_PREFIX; static { // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX. // FIRST_META_REGIONINFO == '.META.,,1'. META_REGION_PREFIX == '.META.,' @@ -60,48 +62,6 @@ public class MetaReader { META_REGION_PREFIX, 0, len); } - /** - * @param ct - * @param tableName A user tablename or a .META. table name. - * @return Interface on to server hosting the -ROOT- or - * .META. regions. - * @throws NotAllMetaRegionsOnlineException - * @throws IOException - */ - private static HRegionInterface getCatalogRegionInterface(final CatalogTracker ct, - final byte [] tableName) - throws NotAllMetaRegionsOnlineException, IOException { - return Bytes.equals(HConstants.META_TABLE_NAME, tableName)? - ct.waitForRootServerConnectionDefault(): - ct.waitForMetaServerConnectionDefault(); - } - - /** - * @param tableName - * @return Returns region name to look in for regions for tableName; - * e.g. if we are looking for .META. regions, we need to look - * in the -ROOT- region, else if a user table, we need to look - * in the .META. region. - */ - private static byte [] getCatalogRegionNameForTable(final byte [] tableName) { - return Bytes.equals(HConstants.META_TABLE_NAME, tableName)? - HRegionInfo.ROOT_REGIONINFO.getRegionName(): - HRegionInfo.FIRST_META_REGIONINFO.getRegionName(); - } - - /** - * @param regionName - * @return Returns region name to look in for regionName; - * e.g. if we are looking for .META.,,1 region, we need to look - * in -ROOT- region, else if a user region, we need to look - * in the .META.,,1 region. - */ - private static byte [] getCatalogRegionNameForRegion(final byte [] regionName) { - return isMetaRegion(regionName)? - HRegionInfo.ROOT_REGIONINFO.getRegionName(): - HRegionInfo.FIRST_META_REGIONINFO.getRegionName(); - } - /** * @param regionName * @return True if regionName is from .META. table. @@ -117,34 +77,14 @@ public class MetaReader { META_REGION_PREFIX, 0, META_REGION_PREFIX.length); } - /** - * Performs a full scan of .META.. - *

- * Returns a map of every region to it's currently assigned server, according - * to META. If the region does not have an assignment it will have a null - * value in the map. - * - * @return map of regions to their currently assigned server where server is - * a String of <host> ':' <port> - * @throws IOException - */ - public static Map fullScan( - CatalogTracker catalogTracker) - throws IOException { - return fullScan(catalogTracker, new TreeSet()); - } - /** * Performs a full scan of .META., skipping regions from any * tables in the specified set of disabled tables. - *

- * Returns a map of every region to it's currently assigned server, according - * to META. If the region does not have an assignment it will have a null - * value in the map. - * * @param catalogTracker * @param disabledTables set of disabled tables that will not be returned - * @return map of regions to their currently assigned server + * @return Returns a map of every region to it's currently assigned server, + * according to META. If the region does not have an assignment it will have + * a null value in the map. * @throws IOException */ public static Map fullScan( @@ -156,16 +96,13 @@ public class MetaReader { /** * Performs a full scan of .META., skipping regions from any * tables in the specified set of disabled tables. - *

- * Returns a map of every region to it's currently assigned server, according - * to META. If the region does not have an assignment it will have a null - * value in the map. - * * @param catalogTracker * @param disabledTables set of disabled tables that will not be returned * @param excludeOfflinedSplitParents If true, do not include offlined split * parents in the return. - * @return map of regions to their currently assigned server + * @return Returns a map of every region to it's currently assigned server, + * according to META. If the region does not have an assignment it will have + * a null value in the map. * @throws IOException */ public static Map fullScan( @@ -178,7 +115,7 @@ public class MetaReader { @Override public boolean visit(Result r) throws IOException { if (r == null || r.isEmpty()) return true; - Pair region = metaRowToRegionPair(r); + Pair region = parseCatalogResult(r); if (region == null) return true; HRegionInfo hri = region.getFirst(); if (disabledTables.contains( @@ -195,38 +132,32 @@ public class MetaReader { /** * Performs a full scan of .META.. - *

- * Returns a map of every region to it's currently assigned server, according - * to META. If the region does not have an assignment it will have a null - * value in the map. - * - * @return map of regions to their currently assigned server + * @return List of {@link Result} * @throws IOException */ - public static List fullScanOfResults( - CatalogTracker catalogTracker) + public static List fullScan(CatalogTracker catalogTracker) throws IOException { - final List regions = new ArrayList(); - Visitor v = new Visitor() { - @Override - public boolean visit(Result r) throws IOException { - if (r == null || r.isEmpty()) return true; - regions.add(r); - return true; - } - }; - fullScan(catalogTracker, v); - return regions; + CollectAllVisitor v = new CollectAllVisitor(); + fullScan(catalogTracker, v, null); + return v.getResults(); + } + + /** + * Performs a full scan of a -ROOT- table. + * @return List of {@link Result} + * @throws IOException + */ + public static List fullScanOfRoot(CatalogTracker catalogTracker) + throws IOException { + CollectAllVisitor v = new CollectAllVisitor(); + fullScan(catalogTracker, v, null, true); + return v.getResults(); } /** * Performs a full scan of .META.. - *

- * Returns a map of every region to it's currently assigned server, according - * to META. If the region does not have an assignment it will have a null - * value in the map. * @param catalogTracker - * @param visitor + * @param visitor Visitor invoked against each row. * @throws IOException */ public static void fullScan(CatalogTracker catalogTracker, @@ -237,24 +168,82 @@ public class MetaReader { /** * Performs a full scan of .META.. - *

- * Returns a map of every region to it's currently assigned server, according - * to META. If the region does not have an assignment it will have a null - * value in the map. * @param catalogTracker - * @param visitor + * @param visitor Visitor invoked against each row. * @param startrow Where to start the scan. Pass null if want to begin scan - * at first row. + * at first row (The visitor will stop the Scan when its done so no need to + * pass a stoprow). * @throws IOException */ public static void fullScan(CatalogTracker catalogTracker, final Visitor visitor, final byte [] startrow) throws IOException { - HRegionInterface metaServer = - catalogTracker.waitForMetaServerConnectionDefault(); - fullScan(metaServer, visitor, - HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), startrow); - return; + fullScan(catalogTracker, visitor, startrow, false); + } + + /** + * Callers should call close on the returned {@link HTable} instance. + * @param catalogTracker We'll use this catalogtracker's connection + * @param tableName Table to get an {@link HTable} against. + * @return An {@link HTable} for tableName + * @throws IOException + */ + private static HTable getHTable(final CatalogTracker catalogTracker, + final byte [] tableName) + throws IOException { + // Passing the CatalogTracker's connection configuration ensures this + // HTable instance uses the CatalogTracker's connection. + return new HTable(catalogTracker.getConnection().getConfiguration(), tableName); + } + + /** + * Callers should call close on the returned {@link HTable} instance. + * @param catalogTracker + * @param regionName + * @return + * @throws IOException + */ + static HTable getCatalogHTable(final CatalogTracker catalogTracker, + final byte [] regionName) + throws IOException { + return isMetaRegion(regionName)? + getRootHTable(catalogTracker): + getMetaHTable(catalogTracker); + } + + /** + * Callers should call close on the returned {@link HTable} instance. + * @param ct + * @return An {@link HTable} for .META. + * @throws IOException + */ + static HTable getMetaHTable(final CatalogTracker ct) + throws IOException { + return getHTable(ct, HConstants.META_TABLE_NAME); + } + + /** + * Callers should call close on the returned {@link HTable} instance. + * @param ct + * @return An {@link HTable} for -ROOT- + * @throws IOException + */ + static HTable getRootHTable(final CatalogTracker ct) + throws IOException { + return getHTable(ct, HConstants.ROOT_TABLE_NAME); + } + + /** + * @param t Table to use (will be closed when done). + * @param g Get to run + * @throws IOException + */ + private static Result get(final HTable t, final Get g) throws IOException { + try { + return t.get(g); + } finally { + t.close(); + } } /** @@ -262,28 +251,29 @@ public class MetaReader { * @param metaServer connection to server hosting ROOT * @return location of META in ROOT where location, or null if not available * @throws IOException + * @deprecated Does not retry; use {@link #readRegionLocation(CatalogTracker, byte[]) */ public static ServerName readMetaLocation(HRegionInterface metaServer) throws IOException { - return readLocation(metaServer, CatalogTracker.ROOT_REGION, - CatalogTracker.META_REGION); + return readLocation(metaServer, CatalogTracker.ROOT_REGION_NAME, + CatalogTracker.META_REGION_NAME); } /** - * Reads the location of the specified region from META. + * Reads the location of the specified region * @param catalogTracker - * @param regionName region to read location of - * @return location of META in ROOT where location is, or null if not available + * @param regionName region whose location we are after + * @return location of region as a {@link ServerName} or null if not found * @throws IOException */ public static ServerName readRegionLocation(CatalogTracker catalogTracker, byte [] regionName) throws IOException { - if (isMetaRegion(regionName)) throw new IllegalArgumentException("See readMetaLocation"); - return readLocation(catalogTracker.waitForMetaServerConnectionDefault(), - CatalogTracker.META_REGION, regionName); + Pair pair = getRegion(catalogTracker, regionName); + return (pair == null || pair.getSecond() == null)? null: pair.getSecond(); } + // TODO: Remove when deprecated dependencies are removed. private static ServerName readLocation(HRegionInterface metaServer, byte [] catalogRegionName, byte [] regionName) throws IOException { @@ -324,15 +314,14 @@ public class MetaReader { if (r == null || r.isEmpty()) { return null; } - return getServerNameFromResult(r); + return getServerNameFromCatalogResult(r); } /** - * Gets the region info and assignment for the specified region from META. + * Gets the region info and assignment for the specified region. * @param catalogTracker - * @param regionName - * @return location of META in ROOT where location is - * a String of <host> ':' <port>, or null if not available + * @param regionName Region to lookup. + * @return Location and HRegionInfo for regionName * @throws IOException */ public static Pair getRegion( @@ -340,41 +329,60 @@ public class MetaReader { throws IOException { Get get = new Get(regionName); get.addFamily(HConstants.CATALOG_FAMILY); - byte [] meta = getCatalogRegionNameForRegion(regionName); - Result r = catalogTracker.waitForMetaServerConnectionDefault().get(meta, get); - return (r == null || r.isEmpty())? null: metaRowToRegionPair(r); + Result r = get(getCatalogHTable(catalogTracker, regionName), get); + return (r == null || r.isEmpty())? null: parseCatalogResult(r); } /** - * @param data A .META. table row. - * @return A pair of the regioninfo and the ServerName + * Extract a {@link ServerName} + * For use on catalog table {@link Result}. + * @param r Result to pull from + * @return A ServerName instance or null if necessary fields not found or empty. + */ + public static ServerName getServerNameFromCatalogResult(final Result r) { + byte[] value = r.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + if (value == null || value.length == 0) return null; + String hostAndPort = Bytes.toString(value); + value = r.getValue(HConstants.CATALOG_FAMILY, + HConstants.STARTCODE_QUALIFIER); + if (value == null || value.length == 0) return null; + return new ServerName(hostAndPort, Bytes.toLong(value)); + } + + /** + * Extract a HRegionInfo and ServerName. + * For use on catalog table {@link Result}. + * @param r Result to pull from + * @return A pair of the {@link HRegionInfo} and the {@link ServerName} * (or null for server address if no address set in .META.). * @throws IOException */ - public static Pair metaRowToRegionPair(Result data) + public static Pair parseCatalogResult(final Result r) throws IOException { - byte [] bytes = data.getValue(HConstants.CATALOG_FAMILY, - HConstants.REGIONINFO_QUALIFIER); - if (bytes == null) return null; - HRegionInfo info = Writables.getHRegionInfo(bytes); - ServerName sn = getServerNameFromResult(data); - // sn can be null in case where no server inof. + HRegionInfo info = + parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER); + ServerName sn = getServerNameFromCatalogResult(r); return new Pair(info, sn); } /** - * @param data Result to interrogate. - * @return A ServerName instance or null if necessary fields not found or empty. + * Parse the content of the cell at {@link HConstants#CATALOG_FAMILY} and + * qualifier as an HRegionInfo and return it, or null. + * For use on catalog table {@link Result}. + * @param r Result instance to pull from. + * @param qualifier Column family qualifier -- either + * {@link HConstants#SPLITA_QUALIFIER}, {@link HConstants#SPLITB_QUALIFIER} or + * {@link HConstants#REGIONINFO_QUALIFIER}. + * @return An HRegionInfo instance or null. + * @throws IOException */ - private static ServerName getServerNameFromResult(final Result data) { - byte[] value = data.getValue(HConstants.CATALOG_FAMILY, - HConstants.SERVER_QUALIFIER); - if (value == null || value.length == 0) return null; - String hostAndPort = Bytes.toString(value); - value = data.getValue(HConstants.CATALOG_FAMILY, - HConstants.STARTCODE_QUALIFIER); - if (value == null || value.length == 0) return null; - return new ServerName(hostAndPort, Bytes.toLong(value)); + public static HRegionInfo parseHRegionInfoFromCatalogResult(final Result r, + byte [] qualifier) + throws IOException { + byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier); + if (bytes == null || bytes.length <= 0) return null; + return Writables.getHRegionInfoOrNull(bytes); } /** @@ -393,21 +401,36 @@ public class MetaReader { // Catalog tables always exist. return true; } - HRegionInterface metaServer = - catalogTracker.waitForMetaServerConnectionDefault(); - Scan scan = getScanForTableName(Bytes.toBytes(tableName)); - scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); - long scannerid = metaServer.openScanner( - HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan); - try { - Result data = metaServer.next(scannerid); - if (data != null && data.size() > 0) { + final byte [] tableNameBytes = Bytes.toBytes(tableName); + // Make a version of ResultCollectingVisitor that only collects the first + CollectingVisitor visitor = new CollectingVisitor() { + private HRegionInfo current = null; + + @Override + public boolean visit(Result r) throws IOException { + this.current = + parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER); + if (this.current == null) { + LOG.warn("No serialized HRegionInfo in " + r); return true; + } + if (!isInsideTable(this.current, tableNameBytes)) return false; + if (this.current.isSplitParent()) return true; + // Else call super and add this Result to the collection. + super.visit(r); + // Stop collecting regions from table after we get one. + return false; } - return false; - } finally { - metaServer.close(scannerid); - } + + @Override + void add(Result r) { + // Add the current HRI. + this.results.add(this.current); + } + }; + fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableNameBytes)); + // If visitor has results >= 1 then table exists. + return visitor.getResults().size() >= 1; } /** @@ -435,42 +458,46 @@ public class MetaReader { public static List getTableRegions(CatalogTracker catalogTracker, byte [] tableName, final boolean excludeOfflinedSplitParents) throws IOException { - if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { - // If root, do a bit of special handling. - List list = new ArrayList(); - list.add(HRegionInfo.ROOT_REGIONINFO); - return list; - } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { - // Same for .META. table - List list = new ArrayList(); - list.add(HRegionInfo.FIRST_META_REGIONINFO); - return list; - } - - // Its a user table. - HRegionInterface metaServer = - getCatalogRegionInterface(catalogTracker, tableName); - List regions = new ArrayList(); - - Scan scan = getScanForTableName(tableName); - scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); - long scannerid = - metaServer.openScanner(getCatalogRegionNameForTable(tableName), scan); + List> result = null; try { - Result data; - while((data = metaServer.next(scannerid)) != null) { - if (data != null && data.size() > 0) { - HRegionInfo info = Writables.getHRegionInfo( - data.getValue(HConstants.CATALOG_FAMILY, - HConstants.REGIONINFO_QUALIFIER)); - if (excludeOfflinedSplitParents && info.isSplitParent()) continue; - regions.add(info); - } - } - return regions; - } finally { - metaServer.close(scannerid); + result = getTableRegionsAndLocations(catalogTracker, tableName, + excludeOfflinedSplitParents); + } catch (InterruptedException e) { + throw new RuntimeException(e); } + return getListOfHRegionInfos(result); + } + + static List getListOfHRegionInfos(final List> pairs) { + if (pairs == null || pairs.isEmpty()) return null; + List result = new ArrayList(pairs.size()); + for (Pair pair: pairs) { + result.add(pair.getFirst()); + } + return result; + } + + /** + * @param current + * @param tableName + * @return True if current tablename is equal to + * tableName + */ + static boolean isInsideTable(final HRegionInfo current, final byte [] tableName) { + return Bytes.equals(tableName, current.getTableName()); + } + + /** + * @param tableName + * @return Place to start Scan in .META. when passed a + * tableName; returns <tableName&rt; <,&rt; <,&rt; + */ + static byte [] getTableStartRowForMeta(final byte [] tableName) { + byte [] startRow = new byte[tableName.length + 2]; + System.arraycopy(tableName, 0, startRow, 0, tableName.length); + startRow[startRow.length - 2] = HRegionInfo.DELIMITER; + startRow[startRow.length - 1] = HRegionInfo.DELIMITER; + return startRow; } /** @@ -504,8 +531,22 @@ public class MetaReader { public static List> getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName) throws IOException, InterruptedException { - byte [] tableNameBytes = Bytes.toBytes(tableName); - if (Bytes.equals(tableNameBytes, HConstants.ROOT_TABLE_NAME)) { + return getTableRegionsAndLocations(catalogTracker, Bytes.toBytes(tableName), + true); + } + + /** + * @param catalogTracker + * @param tableName + * @return Return list of regioninfos and server addresses. + * @throws IOException + * @throws InterruptedException + */ + public static List> + getTableRegionsAndLocations(final CatalogTracker catalogTracker, + final byte [] tableName, final boolean excludeOfflinedSplitParents) + throws IOException, InterruptedException { + if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { // If root, do a bit of special handling. ServerName serverName = catalogTracker.getRootLocation(); List> list = @@ -514,27 +555,36 @@ public class MetaReader { serverName)); return list; } - HRegionInterface metaServer = - getCatalogRegionInterface(catalogTracker, tableNameBytes); - List> regions = - new ArrayList>(); - Scan scan = getScanForTableName(tableNameBytes); - scan.addFamily(HConstants.CATALOG_FAMILY); - long scannerid = - metaServer.openScanner(getCatalogRegionNameForTable(tableNameBytes), scan); - try { - Result data; - while((data = metaServer.next(scannerid)) != null) { - if (data != null && data.size() > 0) { - Pair region = metaRowToRegionPair(data); - if (region == null) continue; - regions.add(region); + // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress + CollectingVisitor> visitor = + new CollectingVisitor>() { + private Pair current = null; + + @Override + public boolean visit(Result r) throws IOException { + HRegionInfo hri = + parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER); + if (hri == null) { + LOG.warn("No serialized HRegionInfo in " + r); + return true; } + if (!isInsideTable(hri, tableName)) return false; + if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; + ServerName sn = getServerNameFromCatalogResult(r); + // Populate this.current so available when we call #add + this.current = new Pair(hri, sn); + // Else call super and add this Result to the collection. + return super.visit(r); } - return regions; - } finally { - metaServer.close(scannerid); - } + + @Override + void add(Result r) { + this.results.add(this.current); + } + }; + fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName), + Bytes.equals(tableName, HConstants.META_TABLE_NAME)); + return visitor.getResults(); } /** @@ -547,36 +597,31 @@ public class MetaReader { public static NavigableMap getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName) throws IOException { - HRegionInterface metaServer = - catalogTracker.waitForMetaServerConnectionDefault(); - NavigableMap hris = new TreeMap(); - Scan scan = new Scan(); - scan.addFamily(HConstants.CATALOG_FAMILY); - long scannerid = metaServer.openScanner( - HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan); - try { - Result result; - while((result = metaServer.next(scannerid)) != null) { - if (result != null && result.size() > 0) { - Pair pair = metaRowToRegionPair(result); - if (pair == null) continue; - if (pair.getSecond() == null || !serverName.equals(pair.getSecond())) { - continue; - } - hris.put(pair.getFirst(), result); - } + final NavigableMap hris = new TreeMap(); + // Fill the above hris map with entries from .META. that have the passed + // servername. + CollectingVisitor v = new CollectingVisitor() { + @Override + void add(Result r) { + if (r == null || r.isEmpty()) return; + ServerName sn = getServerNameFromCatalogResult(r); + if (sn != null && sn.equals(serverName)) this.results.add(r); + } + }; + fullScan(catalogTracker, v); + List results = v.getResults(); + if (results != null && !results.isEmpty()) { + // Convert results to Map keyed by HRI + for (Result r: results) { + Pair p = parseCatalogResult(r); + if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r); } - return hris; - } finally { - metaServer.close(scannerid); } + return hris; } - public static void fullScanMetaAndPrint( - CatalogTracker catalogTracker) + public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker) throws IOException { - final List regions = - new ArrayList(); Visitor v = new Visitor() { @Override public boolean visit(Result r) throws IOException { @@ -590,27 +635,6 @@ public class MetaReader { fullScan(catalogTracker, v); } - - public static List fullScanMetaAndPrintHRIM( - CatalogTracker catalogTracker) - throws IOException { - final List regions = - new ArrayList(); - Visitor v = new Visitor() { - @Override - public boolean visit(Result r) throws IOException { - if (r == null || r.isEmpty()) return true; - LOG.info("fullScanMetaAndPrint1.Current Meta Result: " + r); - HRegionInfo090x hrim = MetaEditor.getHRegionInfoForMigration(r); - LOG.info("fullScanMetaAndPrint.HRIM Print= " + hrim); - regions.add(hrim); - return true; - } - }; - fullScan(catalogTracker, v); - return regions; - } - /** * Fully scan a given region, on a given server starting with given row. * @param hRegionInterface region server @@ -618,6 +642,8 @@ public class MetaReader { * @param regionName name of region * @param startrow start row * @throws IOException + * @deprecated Does not retry; use fullScan xxx instead. + x */ public static void fullScan(HRegionInterface hRegionInterface, Visitor visitor, final byte[] regionName, @@ -638,6 +664,38 @@ public class MetaReader { return; } + /** + * Performs a full scan of a catalog table. + * @param catalogTracker + * @param visitor Visitor invoked against each row. + * @param startrow Where to start the scan. Pass null if want to begin scan + * at first row. + * @param scanRoot True if we are to scan -ROOT- rather than + * .META., the default (pass false to scan .META.) + * @throws IOException + */ + static void fullScan(CatalogTracker catalogTracker, + final Visitor visitor, final byte [] startrow, final boolean scanRoot) + throws IOException { + Scan scan = new Scan(); + if (startrow != null) scan.setStartRow(startrow); + scan.addFamily(HConstants.CATALOG_FAMILY); + HTable metaTable = scanRoot? + getRootHTable(catalogTracker): getMetaHTable(catalogTracker); + ResultScanner scanner = metaTable.getScanner(scan); + try { + Result data; + while((data = scanner.next()) != null) { + if (data.isEmpty()) continue; + // Break if visit returns false. + if (!visitor.visit(data)) break; + } + } finally { + scanner.close(); + metaTable.close(); + } + return; + } /** * Implementations 'visit' a catalog table row. @@ -651,4 +709,37 @@ public class MetaReader { */ public boolean visit(final Result r) throws IOException; } -} + + /** + * A {@link Visitor} that collects content out of passed {@link Result}. + */ + static abstract class CollectingVisitor implements Visitor { + final List results = new ArrayList(); + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + add(r); + return true; + } + + abstract void add(Result r); + + /** + * @return Collected results; wait till visits complete to collect all + * possible results + */ + List getResults() { + return this.results; + } + } + + /** + * Collects all returned. + */ + static class CollectAllVisitor extends CollectingVisitor { + @Override + void add(Result r) { + this.results.add(r); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index a55a4b10811..92c959c1e8c 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -54,7 +54,6 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.ipc.HMasterInterface; import org.apache.hadoop.hbase.ipc.HRegionInterface; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; @@ -138,7 +137,6 @@ public class HBaseAdmin implements Abortable, Closeable { CatalogTracker ct = null; try { ct = new CatalogTracker(this.conf); - ct.start(); } catch (InterruptedException e) { // Let it out as an IOE for now until we redo all so tolerate IEs @@ -1597,19 +1595,19 @@ public class HBaseAdmin implements Abortable, Closeable { * get the regions of a given table. * * @param tableName the name of the table - * @return Ordered list of {@link HRegionInfo}. * + * @return Ordered list of {@link HRegionInfo}. * @throws IOException */ public List getTableRegions(final byte[] tableName) throws IOException { CatalogTracker ct = getCatalogTracker(); - List Regions; + List Regions = null; try { Regions = MetaReader.getTableRegions(ct, tableName, true); } finally { cleanupCatalogTracker(ct); } - return Regions; + return Regions; } public void close() throws IOException { diff --git a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java index 5afaedf2ce7..406e5e34549 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ b/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -130,8 +130,10 @@ import org.apache.zookeeper.KeeperException; */ @SuppressWarnings("serial") public class HConnectionManager { - // A LRU Map of HConnectionKey -> HConnection (TableServer). - private static final Map HBASE_INSTANCES; + // An LRU Map of HConnectionKey -> HConnection (TableServer). All + // access must be synchronized. This map is not private because tests + // need to be able to tinker with it. + static final Map HBASE_INSTANCES; public static final int MAX_CACHED_HBASE_INSTANCES; @@ -895,7 +897,7 @@ public class HConnectionManager { deleteCachedLocation(tableName, row); } - // Query the root or meta region for the location of the meta region + // Query the root or meta region for the location of the meta region regionInfoRow = server.getClosestRowBefore( metaLocation.getRegionInfo().getRegionName(), metaKey, HConstants.CATALOG_FAMILY); @@ -962,8 +964,8 @@ public class HConnectionManager { if (LOG.isDebugEnabled()) { LOG.debug("locateRegionInMeta parentTable=" + Bytes.toString(parentTable) + ", metaLocation=" + - ((metaLocation == null)? "null": metaLocation) + ", attempt=" + - tries + " of " + + ((metaLocation == null)? "null": "{" + metaLocation + "}") + + ", attempt=" + tries + " of " + this.numRetries + " failed; retrying after sleep of " + getPauseTime(tries) + " because: " + e.getMessage()); } @@ -1201,7 +1203,7 @@ public class HConnectionManager { } catch (RemoteException e) { LOG.warn("RemoteException connecting to RS", e); // Throw what the RemoteException was carrying. - throw RemoteExceptionHandler.decodeRemoteException(e); + throw e.unwrapRemoteException(); } } } @@ -1233,19 +1235,22 @@ public class HConnectionManager { public T getRegionServerWithRetries(ServerCallable callable) throws IOException, RuntimeException { - List exceptions = new ArrayList(); + List exceptions = + new ArrayList(); for(int tries = 0; tries < numRetries; tries++) { try { - callable.instantiateServer(tries != 0); callable.beforeCall(); + callable.connect(tries != 0); return callable.call(); } catch (Throwable t) { callable.shouldRetry(t); t = translateException(t); - exceptions.add(t); + RetriesExhaustedException.ThrowableWithExtraContext qt = + new RetriesExhaustedException.ThrowableWithExtraContext(t, + System.currentTimeMillis(), callable.toString()); + exceptions.add(qt); if (tries == numRetries - 1) { - throw new RetriesExhaustedException(callable.getServerName(), - callable.getRegionName(), callable.getRow(), tries, exceptions); + throw new RetriesExhaustedException(tries, exceptions); } } finally { callable.afterCall(); @@ -1254,7 +1259,7 @@ public class HConnectionManager { Thread.sleep(getPauseTime(tries)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new IOException("Giving up trying to get region server: thread is interrupted."); + throw new IOException("Giving up after tries=" + tries, e); } } return null; @@ -1263,8 +1268,8 @@ public class HConnectionManager { public T getRegionServerWithoutRetries(ServerCallable callable) throws IOException, RuntimeException { try { - callable.instantiateServer(false); callable.beforeCall(); + callable.connect(false); return callable.call(); } catch (Throwable t) { Throwable t2 = translateException(t); @@ -1289,7 +1294,7 @@ public class HConnectionManager { return server.multi(multi); } @Override - public void instantiateServer(boolean reload) throws IOException { + public void connect(boolean reload) throws IOException { server = connection.getHRegionConnection(loc.getHostname(), loc.getPort()); } @@ -1804,6 +1809,25 @@ public class HConnectionManager { } return hTableDescriptor; } + } + /** + * Set the number of retries to use serverside when trying to communicate + * with another server over {@link HConnection}. Used updating catalog + * tables, etc. Call this method before we create any Connections. + * @param c The Configuration instance to set the retries into. + * @param log Used to log what we set in here. + */ + public static void setServerSideHConnectionRetries(final Configuration c, + final Log log) { + int hcRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + // Go big. Multiply by 10. If we can't get to meta after this many retries + // then something seriously wrong. + int serversideMultiplier = + c.getInt("hbase.client.serverside.retries.multiplier", 10); + int retries = hcRetries * serversideMultiplier; + c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, retries); + log.debug("Set serverside HConnection retries=" + retries); } } diff --git a/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/src/main/java/org/apache/hadoop/hbase/client/HTable.java index c754064f8c5..fadf217de62 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -53,7 +53,6 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.UnknownScannerException; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; @@ -194,7 +193,8 @@ public class HTable implements HTableInterface, Closeable { } this.connection = HConnectionManager.getConnection(conf); this.scannerTimeout = - (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD); + (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, + HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD); this.operationTimeout = HTableDescriptor.isMetaTable(tableName) ? HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT : conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); @@ -418,8 +418,9 @@ public class HTable implements HTableInterface, Closeable { } }; MetaScanner.metaScan(configuration, visitor, this.tableName); - return new Pair(startKeyList.toArray(new byte[startKeyList.size()][]), - endKeyList.toArray(new byte[endKeyList.size()][])); + return new Pair( + startKeyList.toArray(new byte[startKeyList.size()][]), + endKeyList.toArray(new byte[endKeyList.size()][])); } /** @@ -872,7 +873,7 @@ public class HTable implements HTableInterface, Closeable { @Override public void flushCommits() throws IOException { try { - connection.processBatchOfPuts(writeBuffer, tableName, pool); + this.connection.processBatchOfPuts(writeBuffer, tableName, pool); } finally { if (clearBufferOnFail) { writeBuffer.clear(); diff --git a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java index c8099459773..4135e55d53a 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java +++ b/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java @@ -187,7 +187,7 @@ public class MetaScanner { if (LOG.isDebugEnabled()) { LOG.debug("Scanning " + Bytes.toString(metaTableName) + " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" + - rowUpperLimit + " rows"); + rowUpperLimit + " rows using " + connection.toString()); } callable = new ScannerCallable(connection, metaTableName, scan, null); // Open scanner diff --git a/src/main/java/org/apache/hadoop/hbase/client/Result.java b/src/main/java/org/apache/hadoop/hbase/client/Result.java index 8a0c1a96fb4..bfb8ad750ff 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/Result.java +++ b/src/main/java/org/apache/hadoop/hbase/client/Result.java @@ -20,12 +20,16 @@ package org.apache.hadoop.hbase.client; -import com.google.common.collect.Ordering; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.SplitKeyValue; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.WritableWithSize; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.Writable; import java.io.DataInput; diff --git a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java index 89d2abe6d98..b9042f623f6 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java +++ b/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java @@ -15,9 +15,8 @@ */ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.hbase.util.Bytes; - import java.io.IOException; +import java.util.Date; import java.util.List; /** @@ -35,29 +34,54 @@ public class RetriesExhaustedException extends IOException { super(msg, e); } + /** + * Datastructure that allows adding more info around Throwable incident. + */ + public static class ThrowableWithExtraContext { + private final Throwable t; + private final long when; + private final String extras; + + public ThrowableWithExtraContext(final Throwable t, final long when, + final String extras) { + this.t = t; + this.when = when; + this.extras = extras; + } + + @Override + public String toString() { + return new Date(this.when).toString() + ", " + extras + ", " + t.toString(); + } + } + /** * Create a new RetriesExhaustedException from the list of prior failures. - * @param serverName name of HRegionServer - * @param regionName name of region - * @param row The row we were pursuing when we ran out of retries + * @param callableVitals Details from the {@link ServerCallable} we were using + * when we got this exception. * @param numTries The number of tries we made * @param exceptions List of exceptions that failed before giving up */ - public RetriesExhaustedException(String serverName, final byte [] regionName, - final byte [] row, int numTries, List exceptions) { - super(getMessage(serverName, regionName, row, numTries, exceptions)); + public RetriesExhaustedException(final String callableVitals, int numTries, + List exceptions) { + super(getMessage(callableVitals, numTries, exceptions)); } - private static String getMessage(String serverName, final byte [] regionName, - final byte [] row, - int numTries, List exceptions) { - StringBuilder buffer = new StringBuilder("Trying to contact region server "); - buffer.append(serverName); - buffer.append(" for region "); - buffer.append(regionName == null? "": Bytes.toStringBinary(regionName)); - buffer.append(", row '"); - buffer.append(row == null? "": Bytes.toStringBinary(row)); - buffer.append("', but failed after "); + /** + * Create a new RetriesExhaustedException from the list of prior failures. + * @param numTries + * @param exceptions List of exceptions that failed before giving up + */ + public RetriesExhaustedException(final int numTries, + final List exceptions) { + super(getMessage(numTries, exceptions)); + } + + private static String getMessage(String callableVitals, int numTries, + List exceptions) { + StringBuilder buffer = new StringBuilder("Failed contacting "); + buffer.append(callableVitals); + buffer.append(" after "); buffer.append(numTries + 1); buffer.append(" attempts.\nExceptions:\n"); for (Throwable t : exceptions) { @@ -66,4 +90,16 @@ public class RetriesExhaustedException extends IOException { } return buffer.toString(); } + + private static String getMessage(final int numTries, + final List exceptions) { + StringBuilder buffer = new StringBuilder("Failed after attempts="); + buffer.append(numTries + 1); + buffer.append(", exceptions:\n"); + for (ThrowableWithExtraContext t : exceptions) { + buffer.append(t.toString()); + buffer.append("\n"); + } + return buffer.toString(); + } } \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java index 417ec6cfbdd..4a4679c1435 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java +++ b/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -67,9 +67,9 @@ public class ScannerCallable extends ServerCallable { * @throws IOException */ @Override - public void instantiateServer(boolean reload) throws IOException { + public void connect(boolean reload) throws IOException { if (!instantiated || reload) { - super.instantiateServer(reload); + super.connect(reload); checkIfRegionServerIsRemote(); instantiated = true; } diff --git a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java index 816f8b72767..9b568e3a68f 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java +++ b/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java @@ -31,7 +31,14 @@ import org.apache.hadoop.hbase.ipc.HRegionInterface; import org.apache.hadoop.hbase.util.Bytes; /** - * Abstract class that implements Callable, used by retryable actions. + * Abstract class that implements {@link Callable}. Implementation stipulates + * return type and method we actually invoke on remote Server. Usually + * used inside a try/catch that fields usual connection failures all wrapped + * up in a retry loop. + *

Call {@link #connect(boolean)} to connect to server hosting region + * that contains the passed row in the passed table before invoking + * {@link #call()}. + * @see HConnection#getRegionServerWithoutRetries(ServerCallable) * @param the class that the ServerCallable handles */ public abstract class ServerCallable implements Callable { @@ -44,9 +51,9 @@ public abstract class ServerCallable implements Callable { protected long startTime, endTime; /** - * @param connection connection callable is on - * @param tableName table name callable is on - * @param row row we are querying + * @param connection Connection to use. + * @param tableName Table name to which row belongs. + * @param row The row we want in tableName. */ public ServerCallable(HConnection connection, byte [] tableName, byte [] row) { this(connection, tableName, row, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); @@ -58,34 +65,37 @@ public abstract class ServerCallable implements Callable { this.row = row; this.callTimeout = callTimeout; } + /** - * - * @param reload set this to true if connection should re-find the region + * Connect to the server hosting region with row from tablename. + * @param reload Set this to true if connection should re-find the region * @throws IOException e */ - public void instantiateServer(boolean reload) throws IOException { + public void connect(final boolean reload) throws IOException { this.location = connection.getRegionLocation(tableName, row, reload); this.server = connection.getHRegionConnection(location.getHostname(), location.getPort()); } - /** @return the server name */ + /** @return the server name + * @deprecated Just use {@link #toString()} instead. + */ public String getServerName() { - if (location == null) { - return null; - } + if (location == null) return null; return location.getHostnamePort(); } - /** @return the region name */ + /** @return the region name + * @deprecated Just use {@link #toString()} instead. + */ public byte[] getRegionName() { - if (location == null) { - return null; - } + if (location == null) return null; return location.getRegionInfo().getRegionName(); } - /** @return the row */ + /** @return the row + * @deprecated Just use {@link #toString()} instead. + */ public byte [] getRow() { return row; } diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java index a069400c276..000f99c4fa3 100644 --- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java +++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java @@ -87,7 +87,7 @@ public interface HMasterInterface extends VersionedProtocol { * @throws IOException */ public Pair getAlterStatus(byte[] tableName) - throws IOException; + throws IOException; /** * Adds a column to the specified table diff --git a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 587cf93e9a6..cda50919596 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -267,11 +267,12 @@ public class AssignmentManager extends ZooKeeperListener { * @param tableName * @return Pair indicating the status of the alter command * @throws IOException + * @throws InterruptedException */ public Pair getReopenStatus(byte[] tableName) - throws IOException { - List hris = MetaReader.getTableRegions( - this.master.getCatalogTracker(), tableName); + throws IOException, InterruptedException { + List hris = + MetaReader.getTableRegions(this.master.getCatalogTracker(), tableName); Integer pending = 0; for(HRegionInfo hri : hris) { if(regionsToReopen.get(hri.getEncodedName()) != null) { @@ -730,7 +731,7 @@ public class AssignmentManager extends ZooKeeperListener { case RS_ZK_REGION_OPENING: // Should see OPENING after we have asked it to OPEN or additional // times after already being in state of OPENING - if(regionState == null || + if (regionState == null || (!regionState.isPendingOpen() && !regionState.isOpening())) { LOG.warn("Received OPENING for region " + prettyPrintedRegionName + @@ -1764,7 +1765,6 @@ public class AssignmentManager extends ZooKeeperListener { // Presume that master has stale data. Presume remote side just split. // Presume that the split message when it comes in will fix up the master's // in memory cluster state. - return; } catch (Throwable t) { if (t instanceof RemoteException) { t = ((RemoteException)t).unwrapRemoteException(); @@ -2082,13 +2082,13 @@ public class AssignmentManager extends ZooKeeperListener { Map>> rebuildUserRegions() throws IOException, KeeperException { // Region assignment from META - List results = MetaReader.fullScanOfResults(this.catalogTracker); + List results = MetaReader.fullScan(this.catalogTracker); // Map of offline servers and their regions to be returned Map>> offlineServers = new TreeMap>>(); // Iterate regions in META for (Result result : results) { - Pair region = MetaReader.metaRowToRegionPair(result); + Pair region = MetaReader.parseCatalogResult(result); if (region == null) continue; HRegionInfo regionInfo = region.getFirst(); ServerName regionLocation = region.getSecond(); diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index f80d232f90e..50b49a6e6e7 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -25,7 +25,6 @@ import java.lang.reflect.InvocationTargetException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -41,21 +40,18 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerLoad; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableDescriptors; -import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.catalog.CatalogTracker; -import org.apache.hadoop.hbase.catalog.MetaEditor; import org.apache.hadoop.hbase.catalog.MetaReader; -import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.MetaScanner; -import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType; @@ -64,7 +60,7 @@ import org.apache.hadoop.hbase.ipc.HBaseServer; import org.apache.hadoop.hbase.ipc.HMasterInterface; import org.apache.hadoop.hbase.ipc.HMasterRegionInterface; import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.handler.CreateTableHandler; import org.apache.hadoop.hbase.master.handler.DeleteTableHandler; import org.apache.hadoop.hbase.master.handler.DisableTableHandler; import org.apache.hadoop.hbase.master.handler.EnableTableHandler; @@ -72,18 +68,14 @@ import org.apache.hadoop.hbase.master.handler.ModifyTableHandler; import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler; import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler; import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler; -import org.apache.hadoop.hbase.master.handler.CreateTableHandler; import org.apache.hadoop.hbase.master.metrics.MasterMetrics; import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.replication.regionserver.Replication; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSTableDescriptors; -import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.InfoServer; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Sleeper; @@ -205,7 +197,8 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { public HMaster(final Configuration conf) throws IOException, KeeperException, InterruptedException { this.conf = conf; - + // Set how many times to retry talking to another server over HConnection. + HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG); // Server to handle client requests. String hostname = DNS.getDefaultHost( conf.get("hbase.master.dns.interface", "default"), @@ -470,7 +463,9 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { // Update meta with new HRI if required. i.e migrate all HRI with HTD to // HRI with out HTD in meta and update the status in ROOT. This must happen // before we assign all user regions or else the assignment will fail. - updateMetaWithNewHRI(); + // TODO: Remove this when we do 0.94. + org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD. + updateMetaWithNewHRI(this); // Fixup assignment manager status status.setStatus("Starting assignment manager"); @@ -500,45 +495,6 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { } } - public boolean isMetaHRIUpdated() - throws IOException { - boolean metaUpdated = false; - Get get = new Get(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); - get.addColumn(HConstants.CATALOG_FAMILY, - HConstants.META_MIGRATION_QUALIFIER); - Result r = - catalogTracker.waitForRootServerConnectionDefault().get( - HRegionInfo.ROOT_REGIONINFO.getRegionName(), get); - if (r != null && r.getBytes() != null) - { - byte[] metaMigrated = r.getValue(HConstants.CATALOG_FAMILY, - HConstants.META_MIGRATION_QUALIFIER); - String migrated = Bytes.toString(metaMigrated); - metaUpdated = new Boolean(migrated).booleanValue(); - } else { - LOG.info("metaUpdated = NULL."); - } - LOG.info("Meta updated status = " + metaUpdated); - return metaUpdated; - } - - - boolean updateMetaWithNewHRI() throws IOException { - if (!isMetaHRIUpdated()) { - LOG.info("Meta has HRI with HTDs. Updating meta now."); - try { - MetaEditor.migrateRootAndMeta(this); - LOG.info("ROOT and Meta updated with new HRI."); - return true; - } catch (IOException e) { - throw new RuntimeException("Update ROOT/Meta with new HRI failed." + - "Master startup aborted."); - } - } - LOG.info("ROOT/Meta already up-to date with new HRI."); - return true; - } - /** * Check -ROOT- and .META. are assigned. If not, * assign them. @@ -1018,6 +974,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { Bytes.equals(tableName, HConstants.META_TABLE_NAME); } + @Override public void deleteTable(final byte [] tableName) throws IOException { if (cpHost != null) { cpHost.preDeleteTable(tableName); @@ -1029,16 +986,13 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { } } - /** - * Get the number of regions of the table that have been updated by the alter. - * - * @return Pair indicating the number of regions updated Pair.getFirst is the - * regions that are yet to be updated Pair.getSecond is the total number - * of regions of the table - */ public Pair getAlterStatus(byte[] tableName) throws IOException { - return this.assignmentManager.getReopenStatus(tableName); + try { + return this.assignmentManager.getReopenStatus(tableName); + } catch (InterruptedException e) { + throw new IOException("Interrupted", e); + } } public void addColumn(byte [] tableName, HColumnDescriptor column) @@ -1123,7 +1077,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { if (data == null || data.size() <= 0) { return true; } - Pair pair = MetaReader.metaRowToRegionPair(data); + Pair pair = MetaReader.parseCatalogResult(data); if (pair == null) { return false; } diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java index 1e5d83cfd5f..78bb5bff445 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java +++ b/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java @@ -111,8 +111,8 @@ public class EnableTableHandler extends EventHandler { boolean done = false; // Get the regions of this table. We're done when all listed // tables are onlined. - List regionsInMeta = - MetaReader.getTableRegions(this.ct, tableName, true); + List regionsInMeta; + regionsInMeta = MetaReader.getTableRegions(this.ct, tableName, true); int countOfRegionsInTable = regionsInMeta.size(); List regions = regionsToAssign(regionsInMeta); int regionsCount = regions.size(); diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java index e70bd83538f..0718479a4b2 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java +++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.master.DeadServer; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Writables; import org.apache.zookeeper.KeeperException; /** @@ -265,14 +264,16 @@ public class ServerShutdownHandler extends EventHandler { LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() + " from list of regions to assign because in RIT" + " region state: " + rit.getState()); - hris.remove(rit.getRegion()); + if (hris != null) hris.remove(rit.getRegion()); } } - LOG.info("Reassigning " + (hris == null? 0: hris.size()) + - " region(s) that " + serverName + - " was carrying (skipping " + regionsInTransition.size() + - " regions(s) that are already in transition)"); + assert regionsInTransition != null; + LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) + + " region(s) that " + (serverName == null? "null": serverName) + + " was carrying (skipping " + + regionsInTransition.size() + + " regions(s) that are already in transition)"); // Iterate regions that were on this server and assign them if (hris != null) { @@ -342,7 +343,8 @@ public class ServerShutdownHandler extends EventHandler { final AssignmentManager assignmentManager, final CatalogTracker catalogTracker) throws IOException { - HRegionInfo daughter = getHRegionInfo(result, qualifier); + HRegionInfo daughter = + MetaReader.parseHRegionInfoFromCatalogResult(result, qualifier); if (daughter == null) return; if (isDaughterMissing(catalogTracker, daughter)) { LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString()); @@ -359,21 +361,6 @@ public class ServerShutdownHandler extends EventHandler { } } - /** - * Interpret the content of the cell at {@link HConstants#CATALOG_FAMILY} and - * qualifier as an HRegionInfo and return it, or null. - * @param r Result instance to pull from. - * @param qualifier Column family qualifier - * @return An HRegionInfo instance or null. - * @throws IOException - */ - private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) - throws IOException { - byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier); - if (bytes == null || bytes.length <= 0) return null; - return Writables.getHRegionInfoOrNull(bytes); - } - /** * Look for presence of the daughter OR of a split of the daughter in .META. * Daughter could have been split over on regionserver before a run of the @@ -416,7 +403,8 @@ public class ServerShutdownHandler extends EventHandler { @Override public boolean visit(Result r) throws IOException { - HRegionInfo hri = getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER); + HRegionInfo hri = + MetaReader.parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER); if (hri == null) { LOG.warn("No serialized HRegionInfo in " + r); return true; diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 78a531d21b9..fcd071af4d7 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -3061,9 +3061,9 @@ public class HRegion implements HeapSize { // , Writable{ EnvironmentEdgeManager.currentTimeMillis(), Writables.getBytes(r.getRegionInfo()))); edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY, - HConstants.META_MIGRATION_QUALIFIER, + org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER, EnvironmentEdgeManager.currentTimeMillis(), - Bytes.toBytes("true"))); + Bytes.toBytes(true))); meta.put(HConstants.CATALOG_FAMILY, edits); } finally { diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 49ab17a5e8e..e2e694a0101 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -83,6 +83,7 @@ import org.apache.hadoop.hbase.client.Action; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.MultiAction; import org.apache.hadoop.hbase.client.MultiPut; @@ -334,6 +335,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, throws IOException, InterruptedException { this.fsOk = true; this.conf = conf; + // Set how many times to retry talking to another server over HConnection. + HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG); this.isOnline = false; checkCodecs(this.conf); diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java index 74748ec6405..7c04a723f51 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java @@ -78,4 +78,4 @@ public interface RegionServerServices extends OnlineRegions { */ public Map getRegionsInTransitionInRS(); -} +} diff --git a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java index 7f21c9f7549..b4b49d57952 100644 --- a/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java +++ b/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java @@ -274,17 +274,19 @@ public class ThriftServer { public List getTableRegions(ByteBuffer tableName) throws IOError { try{ - List HRegions = this.admin.getTableRegions(tableName.array()); + List hris = this.admin.getTableRegions(tableName.array()); List regions = new ArrayList(); - for (HRegionInfo regionInfo : HRegions){ - TRegionInfo region = new TRegionInfo(); - region.startKey = ByteBuffer.wrap(regionInfo.getStartKey()); - region.endKey = ByteBuffer.wrap(regionInfo.getEndKey()); - region.id = regionInfo.getRegionId(); - region.name = ByteBuffer.wrap(regionInfo.getRegionName()); - region.version = regionInfo.getVersion(); - regions.add(region); + if (hris != null) { + for (HRegionInfo regionInfo : hris){ + TRegionInfo region = new TRegionInfo(); + region.startKey = ByteBuffer.wrap(regionInfo.getStartKey()); + region.endKey = ByteBuffer.wrap(regionInfo.getEndKey()); + region.id = regionInfo.getRegionId(); + region.name = ByteBuffer.wrap(regionInfo.getRegionName()); + region.version = regionInfo.getVersion(); + regions.add(region); + } } return regions; } catch (IOException e){ diff --git a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 154ac32708e..7409c9c1c1c 100644 --- a/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -891,8 +891,7 @@ public class HBaseFsck { // record the latest modification of this META record long ts = Collections.max(result.list(), comp).getTimestamp(); - Pair pair = - MetaReader.metaRowToRegionPair(result); + Pair pair = MetaReader.parseCatalogResult(result); if (pair == null || pair.getFirst() == null) { emptyRegionInfoQualifiers.add(result); return true; diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java index 55257b3c461..4da6f962728 100644 --- a/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java @@ -19,52 +19,29 @@ */ package org.apache.hadoop.hbase.zookeeper; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.catalog.CatalogTracker; /** * Tracks the unassigned zookeeper node used by the META table. - * - * A callback is made into the passed {@link CatalogTracker} when - * .META. completes a new assignment. *

* If META is already assigned when instantiating this class, you will not * receive any notification for that assignment. You will receive a * notification after META has been successfully assigned to a new location. */ public class MetaNodeTracker extends ZooKeeperNodeTracker { - private static final Log LOG = LogFactory.getLog(MetaNodeTracker.class); - - /** Catalog tracker to notify when META has a new assignment completed. */ - private final CatalogTracker catalogTracker; - /** * Creates a meta node tracker. * @param watcher * @param abortable */ - public MetaNodeTracker(final ZooKeeperWatcher watcher, - final CatalogTracker catalogTracker, final Abortable abortable) { + public MetaNodeTracker(final ZooKeeperWatcher watcher, final Abortable abortable) { super(watcher, ZKUtil.joinZNode(watcher.assignmentZNode, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()), abortable); - this.catalogTracker = catalogTracker; } @Override public void nodeDeleted(String path) { super.nodeDeleted(path); - if (!path.equals(node)) return; - LOG.info("Detected completed assignment of META, notifying catalog tracker"); - try { - this.catalogTracker.waitForMetaServerConnectionDefault(); - } catch (IOException e) { - LOG.warn("Tried to reset META server location after seeing the " + - "completion of a new META assignment but got an IOE", e); - } } } \ No newline at end of file diff --git a/src/main/ruby/hbase/admin.rb b/src/main/ruby/hbase/admin.rb index b244ffeba3a..56f0b4f6ad8 100644 --- a/src/main/ruby/hbase/admin.rb +++ b/src/main/ruby/hbase/admin.rb @@ -263,8 +263,13 @@ module Hbase #---------------------------------------------------------------------------------------------- # Truncates table (deletes all records by recreating the table) - def truncate(table_name) - h_table = org.apache.hadoop.hbase.client.HTable.new(table_name) + def truncate(table_name, conf = nil) + h_table = nil + unless conf + h_table = org.apache.hadoop.hbase.client.HTable.new(table_name) + else + h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name) + end table_description = h_table.getTableDescriptor() yield 'Disabling table...' if block_given? disable(table_name) diff --git a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java index fc056152dea..cc3d555796c 100644 --- a/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java +++ b/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java @@ -25,7 +25,6 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -81,15 +80,11 @@ public class TestRegionRebalancing { this.table = new HTable(UTIL.getConfiguration(), this.desc.getName()); CatalogTracker ct = new CatalogTracker(UTIL.getConfiguration()); ct.start(); - Map regions = null; try { - regions = MetaReader.fullScan(ct); + MetaReader.fullScanMetaAndPrint(ct); } finally { ct.stop(); } - for (Map.Entry e: regions.entrySet()) { - LOG.info(e); - } assertEquals("Test table should have right number of regions", HBaseTestingUtility.KEYS.length + 1/*One extra to account for start/end keys*/, this.table.getStartKeys().length); diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java index 538e8091886..8e16b0af379 100644 --- a/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java +++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java @@ -35,16 +35,20 @@ import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ServerCallable; import org.apache.hadoop.hbase.ipc.HRegionInterface; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.util.Progressable; @@ -160,27 +164,44 @@ public class TestCatalogTracker { t.join(); } - @Test public void testGetMetaServerConnectionFails() + @Test + public void testGetMetaServerConnectionFails() throws IOException, InterruptedException, KeeperException { - HConnection connection = Mockito.mock(HConnection.class); - ConnectException connectException = - new ConnectException("Connection refused"); - final HRegionInterface implementation = - Mockito.mock(HRegionInterface.class); - Mockito.when(implementation.get((byte [])Mockito.any(), (Get)Mockito.any())). - thenThrow(connectException); - Mockito.when(connection.getHRegionConnection((HServerAddress)Matchers.anyObject(), Matchers.anyBoolean())). - thenReturn(implementation); - Assert.assertNotNull(connection.getHRegionConnection(new HServerAddress(), false)); - final CatalogTracker ct = constructAndStartCatalogTracker(connection); + HConnection connection = + HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration()); try { - RootLocationEditor.setRootLocation(this.watcher, - new ServerName("example.com", 1234, System.currentTimeMillis())); - Assert.assertFalse(ct.verifyMetaRegionLocation(100)); + // Mock an HRegionInterface. + final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + // If a 'get' is called on mocked interface, throw connection refused. + Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())). + thenThrow(new ConnectException("Connection refused")); + // Make it so our implementation is returned when we do a connection. + // Need to fake out the location lookup stuff first. + ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis()); + final HRegionLocation anyLocation = + new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), + sn.getPort()); + Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(), + (byte[]) Mockito.any(), Mockito.anyBoolean())). + thenReturn(anyLocation); + Mockito.when(connection.getHRegionConnection(Mockito.anyString(), + Mockito.anyInt(), Matchers.anyBoolean())). + thenReturn(implementation); + // Now start up the catalogtracker with our doctored Connection. + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + try { + RootLocationEditor.setRootLocation(this.watcher, sn); + long timeout = UTIL.getConfiguration(). + getLong("hbase.catalog.verification.timeout", 1000); + Assert.assertFalse(ct.verifyMetaRegionLocation(timeout)); + } finally { + // Clean out root location or later tests will be confused... they + // presume start fresh in zk. + RootLocationEditor.deleteRootLocation(this.watcher); + } } finally { - // Clean out root location or later tests will be confused... they presume - // start fresh in zk. - RootLocationEditor.deleteRootLocation(this.watcher); + // Clear out our doctored connection or could mess up subsequent tests. + HConnectionManager.deleteConnection(UTIL.getConfiguration(), true); } } @@ -200,9 +221,9 @@ public class TestCatalogTracker { Mockito.mock(HRegionInterface.class); Mockito.when(implementation.getRegionInfo((byte [])Mockito.any())). thenThrow(connectException); - Mockito.when(connection.getHRegionConnection((HServerAddress)Matchers.anyObject(), Matchers.anyBoolean())). + Mockito.when(connection.getHRegionConnection(Mockito.anyString(), + Mockito.anyInt(), Mockito.anyBoolean())). thenReturn(implementation); - Assert.assertNotNull(connection.getHRegionConnection(new HServerAddress(), false)); final CatalogTracker ct = constructAndStartCatalogTracker(connection); try { RootLocationEditor.setRootLocation(this.watcher, @@ -225,8 +246,14 @@ public class TestCatalogTracker { @Test (expected = NotAllMetaRegionsOnlineException.class) public void testTimeoutWaitForMeta() throws IOException, InterruptedException { - final CatalogTracker ct = constructAndStartCatalogTracker(); - ct.waitForMeta(100); + HConnection connection = + HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration()); + try { + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + ct.waitForMeta(100); + } finally { + HConnectionManager.deleteConnection(UTIL.getConfiguration(), true); + } } /** @@ -259,62 +286,84 @@ public class TestCatalogTracker { /** * Test waiting on meta w/ no timeout specified. - * @throws IOException - * @throws InterruptedException - * @throws KeeperException + * @throws Exception */ @Test public void testNoTimeoutWaitForMeta() - throws IOException, InterruptedException, KeeperException { + throws Exception { // Mock an HConnection and a HRegionInterface implementation. Have the // HConnection return the HRI. Have the HRI return a few mocked up responses // to make our test work. - HConnection connection = Mockito.mock(HConnection.class); - HRegionInterface mockHRI = Mockito.mock(HRegionInterface.class); - // Make the HRI return an answer no matter how Get is called. Same for - // getHRegionInfo. Thats enough for this test. - Mockito.when(connection.getHRegionConnection((String)Mockito.any(), - Matchers.anyInt())).thenReturn(mockHRI); + HConnection connection = + HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration()); + try { + // Mock an HRegionInterface. + + final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + // Make it so our implementation is returned when we do a connection. + // Need to fake out the location lookup stuff first. + ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis()); + final HRegionLocation anyLocation = + new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), + sn.getPort()); + Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(), + (byte[]) Mockito.any(), Mockito.anyBoolean())). + thenReturn(anyLocation); + // Have implementation returned which ever way getHRegionConnection is called. + Mockito.when(connection.getHRegionConnection(Mockito.anyString(), + Mockito.anyInt(), Matchers.anyBoolean())). + thenReturn(implementation); + Mockito.when(connection.getHRegionConnection(Mockito.anyString(), + Mockito.anyInt())). + thenReturn(implementation); - final CatalogTracker ct = constructAndStartCatalogTracker(connection); - ServerName hsa = ct.getMetaLocation(); - Assert.assertNull(hsa); + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + ServerName hsa = ct.getMetaLocation(); + Assert.assertNull(hsa); - // Now test waiting on meta location getting set. - Thread t = new WaitOnMetaThread(ct) { - @Override - void doWaiting() throws InterruptedException { - this.ct.waitForMeta(); - } - }; - startWaitAliveThenWaitItLives(t, 1000); + // Now test waiting on meta location getting set. + Thread t = new WaitOnMetaThread(ct) { + @Override + void doWaiting() throws InterruptedException { + this.ct.waitForMeta(); + } + }; + startWaitAliveThenWaitItLives(t, 1000); - // Now the ct is up... set into the mocks some answers that make it look - // like things have been getting assigned. Make it so we'll return a - // location (no matter what the Get is). Same for getHRegionInfo -- always - // just return the meta region. - List kvs = new ArrayList(); - kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, - HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, - Bytes.toBytes(SN.getHostAndPort()))); - kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, - HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, - Bytes.toBytes(SN.getStartcode()))); - final Result result = new Result(kvs); - Mockito.when(mockHRI.get((byte [])Mockito.any(), (Get)Mockito.any())). - thenReturn(result); - Mockito.when(mockHRI.getRegionInfo((byte [])Mockito.any())). - thenReturn(HRegionInfo.FIRST_META_REGIONINFO); - // This should trigger wake up of meta wait (Its the removal of the meta - // region unassigned node that triggers catalogtrackers that a meta has - // been assigned. - String node = ct.getMetaNodeTracker().getNode(); - ZKUtil.createAndFailSilent(this.watcher, node); - MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN); - ZKUtil.deleteNode(this.watcher, node); - // Join the thread... should exit shortly. - t.join(); - // Now meta is available. - Assert.assertTrue(ct.getMetaLocation().equals(SN)); + // Now the ct is up... set into the mocks some answers that make it look + // like things have been getting assigned. Make it so we'll return a + // location (no matter what the Get is). Same for getHRegionInfo -- always + // just return the meta region. + List kvs = new ArrayList(); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO))); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(SN.getHostAndPort()))); + kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY, + HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, + Bytes.toBytes(SN.getStartcode()))); + final Result result = new Result(kvs); + Mockito.when(connection.getRegionServerWithRetries((ServerCallable)Mockito.any())). + thenReturn(result); + Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())). + thenReturn(HRegionInfo.FIRST_META_REGIONINFO); + // This should trigger wake up of meta wait (Its the removal of the meta + // region unassigned node that triggers catalogtrackers that a meta has + // been assigned). + String node = ct.getMetaNodeTracker().getNode(); + ZKUtil.createAndFailSilent(this.watcher, node); + MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN); + ZKUtil.deleteNode(this.watcher, node); + // Go get the new meta location. waitForMeta gets and verifies meta. + Assert.assertTrue(ct.waitForMeta(10000).equals(SN)); + // Join the thread... should exit shortly. + t.join(); + // Now meta is available. + Assert.assertTrue(ct.waitForMeta(10000).equals(SN)); + } finally { + HConnectionManager.deleteConnection(UTIL.getConfiguration(), true); + } } private void startWaitAliveThenWaitItLives(final Thread t, final int ms) { @@ -361,4 +410,4 @@ public class TestCatalogTracker { this.ct.waitForRoot(); } } -} +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java index 84130e2b4b3..5d7bf0cd127 100644 --- a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java +++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java @@ -19,7 +19,10 @@ */ package org.apache.hadoop.hbase.catalog; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.List; @@ -34,8 +37,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -70,11 +71,15 @@ public class TestMetaReaderEditor { }; @BeforeClass public static void beforeClass() throws Exception { - UTIL.startMiniCluster(); + UTIL.startMiniCluster(3); } @Before public void setup() throws IOException, InterruptedException { Configuration c = new Configuration(UTIL.getConfiguration()); + // Tests to 4 retries every 5 seconds. Make it try every 1 second so more + // responsive. 1 second is default as is ten retries. + c.setLong("hbase.client.pause", 1000); + c.setInt("hbase.client.retries.number", 10); zkw = new ZooKeeperWatcher(c, "TestMetaReaderEditor", ABORTABLE); ct = new CatalogTracker(zkw, c, ABORTABLE); ct.start(); @@ -84,6 +89,111 @@ public class TestMetaReaderEditor { UTIL.shutdownMiniCluster(); } + /** + * Does {@link MetaReader#getRegion(CatalogTracker, byte[])} and a write + * against .META. while its hosted server is restarted to prove our retrying + * works. + * @throws IOException + * @throws InterruptedException + */ + @Test (timeout = 180000) public void testRetrying() + throws IOException, InterruptedException { + final String name = "testRetrying"; + LOG.info("Started " + name); + final byte [] nameBytes = Bytes.toBytes(name); + HTable t = UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY); + int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY); + // Test it works getting a region from just made user table. + final List regions = + testGettingTableRegions(this.ct, nameBytes, regionCount); + MetaTask reader = new MetaTask(this.ct, "reader") { + @Override + void metaTask() throws Throwable { + testGetRegion(this.ct, regions.get(0)); + LOG.info("Read " + regions.get(0).getEncodedName()); + } + }; + MetaTask writer = new MetaTask(this.ct, "writer") { + @Override + void metaTask() throws Throwable { + MetaEditor.addRegionToMeta(this.ct, regions.get(0)); + LOG.info("Wrote " + regions.get(0).getEncodedName()); + } + }; + reader.start(); + writer.start(); + // Make sure reader and writer are working. + assertTrue(reader.isProgressing()); + assertTrue(writer.isProgressing()); + // Kill server hosting meta -- twice . See if our reader/writer ride over the + // meta moves. They'll need to retry. + for (int i = 0; i < 2; i++) { + LOG.info("Restart=" + i); + UTIL.ensureSomeRegionServersAvailable(2); + int index = -1; + do { + index = UTIL.getMiniHBaseCluster().getServerWithMeta(); + } while (index == -1); + UTIL.getMiniHBaseCluster().abortRegionServer(index); + UTIL.getMiniHBaseCluster().waitOnRegionServer(index); + } + assertTrue(reader.toString(), reader.isProgressing()); + assertTrue(writer.toString(), writer.isProgressing()); + reader.stop = true; + writer.stop = true; + reader.join(); + writer.join(); + } + + /** + * Thread that runs a MetaReader/MetaEditor task until asked stop. + */ + abstract static class MetaTask extends Thread { + boolean stop = false; + int count = 0; + Throwable t = null; + final CatalogTracker ct; + + MetaTask(final CatalogTracker ct, final String name) { + super(name); + this.ct = ct; + } + + @Override + public void run() { + try { + while(!this.stop) { + LOG.info("Before " + this.getName()+ ", count=" + this.count); + metaTask(); + this.count += 1; + LOG.info("After " + this.getName() + ", count=" + this.count); + Thread.sleep(100); + } + } catch (Throwable t) { + LOG.info(this.getName() + " failed", t); + this.t = t; + } + } + + boolean isProgressing() throws InterruptedException { + int currentCount = this.count; + while(currentCount == this.count) { + if (!isAlive()) return false; + if (this.t != null) return false; + Thread.sleep(10); + } + return true; + } + + @Override + public String toString() { + return "count=" + this.count + ", t=" + + (this.t == null? "null": this.t.toString()); + } + + abstract void metaTask() throws Throwable; + } + @Test public void testGetRegionsCatalogTables() throws IOException, InterruptedException { List regions = @@ -114,19 +224,9 @@ public class TestMetaReaderEditor { @Test public void testGetRegion() throws IOException, InterruptedException { final String name = "testGetRegion"; LOG.info("Started " + name); - final byte [] nameBytes = Bytes.toBytes(name); - HTable t = UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY); - int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY); - - // Test it works getting a region from user table. - List regions = MetaReader.getTableRegions(ct, nameBytes); - assertEquals(regionCount, regions.size()); - Pair pair = - MetaReader.getRegion(ct, regions.get(0).getRegionName()); - assertEquals(regions.get(0).getEncodedName(), - pair.getFirst().getEncodedName()); // Test get on non-existent region. - pair = MetaReader.getRegion(ct, Bytes.toBytes("nonexistent-region")); + Pair pair = + MetaReader.getRegion(ct, Bytes.toBytes("nonexistent-region")); assertNull(pair); // Test it works getting a region from meta/root. pair = @@ -137,7 +237,8 @@ public class TestMetaReaderEditor { } // Test for the optimization made in HBASE-3650 - @Test public void testScanMetaForTable() throws IOException { + @Test public void testScanMetaForTable() + throws IOException, InterruptedException { final String name = "testScanMetaForTable"; LOG.info("Started " + name); @@ -165,4 +266,25 @@ public class TestMetaReaderEditor { } assertEquals(1, MetaReader.getTableRegions(ct, greaterName).size()); } -} \ No newline at end of file + + private static List testGettingTableRegions(final CatalogTracker ct, + final byte [] nameBytes, final int regionCount) + throws IOException, InterruptedException { + List regions = MetaReader.getTableRegions(ct, nameBytes); + assertEquals(regionCount, regions.size()); + Pair pair = + MetaReader.getRegion(ct, regions.get(0).getRegionName()); + assertEquals(regions.get(0).getEncodedName(), + pair.getFirst().getEncodedName()); + return regions; + } + + private static void testGetRegion(final CatalogTracker ct, + final HRegionInfo region) + throws IOException, InterruptedException { + Pair pair = + MetaReader.getRegion(ct, region.getRegionName()); + assertEquals(region.getEncodedName(), + pair.getFirst().getEncodedName()); + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java new file mode 100644 index 00000000000..c97fb3d5ba5 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Test MetaReader/Editor but without spinning up a cluster. + * We mock regionserver back and forth (we do spin up a zk cluster). + */ +public class TestMetaReaderEditorNoCluster { + private static final Log LOG = LogFactory.getLog(TestMetaReaderEditorNoCluster.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final Abortable ABORTABLE = new Abortable() { + boolean aborted = false; + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + this.aborted = true; + throw new RuntimeException(e); + } + @Override + public boolean isAborted() { + return this.aborted; + } + }; + + @Before + public void before() throws Exception { + UTIL.startMiniZKCluster(); + } + + @After + public void after() throws IOException { + UTIL.shutdownMiniZKCluster(); + } + + /** + * Test that MetaReader will ride over server throwing + * "Server not running" IOEs. + * @see https://issues.apache.org/jira/browse/HBASE-3446 + * @throws IOException + * @throws InterruptedException + */ + @Test + public void testRideOverServerNotRunning() throws IOException, InterruptedException { + // Need a zk watcher. + ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(), + this.getClass().getSimpleName(), ABORTABLE, true); + // This is a servername we use in a few places below. + ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis()); + + HConnection connection = null; + CatalogTracker ct = null; + try { + // Mock an HRegionInterface. Our mock implementation will fail a few + // times when we go to open a scanner. + final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); + // When openScanner called throw IOE 'Server not running' a few times + // before we return a scanner id. Whats WEIRD is that these + // exceptions do not show in the log because they are caught and only + // printed if we FAIL. We eventually succeed after retry so these don't + // show. We will know if they happened or not because we will ask + // mockito at the end of this test to verify that openscanner was indeed + // called the wanted number of times. + final long scannerid = 123L; + Mockito.when(implementation.openScanner((byte [])Mockito.any(), + (Scan)Mockito.any())). + thenThrow(new IOException("Server not running (1 of 3)")). + thenThrow(new IOException("Server not running (2 of 3)")). + thenThrow(new IOException("Server not running (3 of 3)")). + thenReturn(scannerid); + // Make it so a verifiable answer comes back when next is called. Return + // the verifiable answer and then a null so we stop scanning. Our + // verifiable answer is something that looks like a row in META with + // a server and startcode that is that of the above defined servername. + List kvs = new ArrayList(); + final byte [] rowToVerify = Bytes.toBytes("rowToVerify"); + kvs.add(new KeyValue(rowToVerify, + HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO))); + kvs.add(new KeyValue(rowToVerify, + HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, + Bytes.toBytes(sn.getHostAndPort()))); + kvs.add(new KeyValue(rowToVerify, + HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, + Bytes.toBytes(sn.getStartcode()))); + final Result [] result = new Result [] {new Result(kvs)}; + Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())). + thenReturn(result). + thenReturn(null); + + // Associate a spied-upon HConnection with UTIL.getConfiguration. Need + // to shove this in here first so it gets picked up all over; e.g. by + // HTable. + connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration()); + // Fix the location lookup so it 'works' though no network. First + // make an 'any location' object. + final HRegionLocation anyLocation = + new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), + sn.getPort()); + // Return the any location object when locateRegion is called in HTable + // constructor and when its called by ServerCallable (it uses getRegionLocation). + // The ugly format below comes of 'Important gotcha on spying real objects!' from + // http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html + Mockito.doReturn(anyLocation). + when(connection).locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any()); + Mockito.doReturn(anyLocation). + when(connection).getRegionLocation((byte[]) Mockito.any(), + (byte[]) Mockito.any(), Mockito.anyBoolean()); + + // Now shove our HRI implementation into the spied-upon connection. + Mockito.doReturn(implementation). + when(connection).getHRegionConnection(Mockito.anyString(), Mockito.anyInt()); + + // Now start up the catalogtracker with our doctored Connection. + ct = new CatalogTracker(zkw, null, connection, ABORTABLE, 0); + ct.start(); + // Scan meta for user tables and verify we got back expected answer. + NavigableMap hris = MetaReader.getServerUserRegions(ct, sn); + assertTrue(hris.size() == 1); + assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO)); + assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow())); + // Finally verify that openscanner was called four times -- three times + // with exception and then on 4th attempt we succeed. + Mockito.verify(implementation, Mockito.times(4)). + openScanner((byte [])Mockito.any(), (Scan)Mockito.any()); + } finally { + if (ct != null) ct.stop(); + HConnectionManager.deleteConnection(UTIL.getConfiguration(), true); + zkw.close(); + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java new file mode 100644 index 00000000000..20a70f1c94d --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation; +import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey; +import org.mockito.Mockito; + +/** + * {@link HConnection} testing utility. + */ +public class HConnectionTestingUtility { + /* + * Not part of {@link HBaseTestingUtility} because this class is not + * in same package as {@link HConnection}. Would have to reveal ugly + * {@link HConnectionManager} innards to HBaseTestingUtility to give it access. + */ + /** + * Get a Mocked {@link HConnection} that goes with the passed conf + * configuration instance. Minimally the mock will return + * conf when {@link HConnection#getConfiguration()} is invoked. + * Be sure to shutdown the connection when done by calling + * {@link HConnectionManager#deleteConnection(Configuration, boolean)} else it + * will stick around; this is probably not what you want. + * @param conf configuration + * @return HConnection object for conf + * @throws ZooKeeperConnectionException + */ + public static HConnection getMockedConnection(final Configuration conf) + throws ZooKeeperConnectionException { + HConnectionKey connectionKey = new HConnectionKey(conf); + synchronized (HConnectionManager.HBASE_INSTANCES) { + HConnectionImplementation connection = + HConnectionManager.HBASE_INSTANCES.get(connectionKey); + if (connection == null) { + connection = Mockito.mock(HConnectionImplementation.class); + Mockito.when(connection.getConfiguration()).thenReturn(conf); + HConnectionManager.HBASE_INSTANCES.put(connectionKey, connection); + } + return connection; + } + } + + /** + * Get a Mockito spied-upon {@link HConnection} that goes with the passed + * conf configuration instance. + * Be sure to shutdown the connection when done by calling + * {@link HConnectionManager#deleteConnection(Configuration, boolean)} else it + * will stick around; this is probably not what you want. + * @param conf configuration + * @return HConnection object for conf + * @throws ZooKeeperConnectionException + * @see http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html#spy(T) + */ + public static HConnection getSpiedConnection(final Configuration conf) + throws ZooKeeperConnectionException { + HConnectionKey connectionKey = new HConnectionKey(conf); + synchronized (HConnectionManager.HBASE_INSTANCES) { + HConnectionImplementation connection = + HConnectionManager.HBASE_INSTANCES.get(connectionKey); + if (connection == null) { + connection = Mockito.spy(new HConnectionImplementation(conf)); + HConnectionManager.HBASE_INSTANCES.put(connectionKey, connection); + } + return connection; + } + } +} diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java index f09944ec85d..f98baeffbf0 100644 --- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java +++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java @@ -19,6 +19,10 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashSet; @@ -27,6 +31,8 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -38,12 +44,6 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** * This class is for testing HCM features @@ -225,4 +225,4 @@ public class TestHCM { Thread.sleep(50); } } -} +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java index 645bca6b83e..6cbf88c22fd 100644 --- a/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java +++ b/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java @@ -28,7 +28,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD; import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.HTableDescriptor; @@ -77,7 +77,8 @@ public class TestMetaMigration { @Test public void testMetaUpdatedFlagInROOT() throws Exception { LOG.info("Starting testMetaUpdatedFlagInROOT"); - boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated(); + boolean metaUpdated = + MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster()); assertEquals(true, metaUpdated); LOG.info("END testMetaUpdatedFlagInROOT"); } @@ -97,16 +98,17 @@ public class TestMetaMigration { Bytes.toBytes("region_b")}); CatalogTracker ct = miniHBaseCluster.getMaster().getCatalogTracker(); // just for this test set it to false. - MetaEditor.updateRootWithMetaMigrationStatus(ct, false); + MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct, false); MetaReader.fullScanMetaAndPrint(ct); - LOG.info("MEta Print completed.testUpdatesOnMetaWithLegacyHRI"); + LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI"); - List htds = MetaEditor.updateMetaWithNewRegionInfo( + List htds = MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo( TEST_UTIL.getHBaseCluster().getMaster()); MetaReader.fullScanMetaAndPrint(ct); assertEquals(3, htds.size()); // Assert that the flag in ROOT is updated to reflect the correct status - boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated(); + boolean metaUpdated = + MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster()); assertEquals(true, metaUpdated); LOG.info("END testMetaWithLegacyHRI"); @@ -138,15 +140,16 @@ public class TestMetaMigration { TEST_UTIL.createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10); CatalogTracker ct = miniHBaseCluster.getMaster().getCatalogTracker(); // just for this test set it to false. - MetaEditor.updateRootWithMetaMigrationStatus(ct, false); + MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct, false); //MetaReader.fullScanMetaAndPrint(ct); LOG.info("MEta Print completed.testUpdatesOnMetaWithLegacyHRI"); - List htds = MetaEditor.updateMetaWithNewRegionInfo( + List htds = MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo( TEST_UTIL.getHBaseCluster().getMaster()); assertEquals(10, htds.size()); // Assert that the flag in ROOT is updated to reflect the correct status - boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated(); + boolean metaUpdated = + MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster()); assertEquals(true, metaUpdated); LOG.info("END testMetaWithLegacyHRI"); diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index 8fcdcccdfc4..2f20596d3c3 100644 --- a/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -47,6 +47,9 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.catalog.CatalogTracker; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.io.Reference; @@ -61,20 +64,27 @@ import org.mockito.Mockito; public class TestCatalogJanitor { /** * Pseudo server for below tests. + * Be sure to call stop on the way out else could leave some mess around. */ class MockServer implements Server { + private final HConnection connection; private final Configuration c; private final CatalogTracker ct; MockServer(final HBaseTestingUtility htu) - throws NotAllMetaRegionsOnlineException, IOException { + throws NotAllMetaRegionsOnlineException, IOException, InterruptedException { this.c = htu.getConfiguration(); + // Mock an HConnection and a HRegionInterface implementation. Have the + // HConnection return the HRI. Have the HRI return a few mocked up responses + // to make our test work. + this.connection = HConnectionTestingUtility.getMockedConnection(this.c); // Set hbase.rootdir into test dir. FileSystem fs = FileSystem.get(this.c); Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR))); this.c.set(HConstants.HBASE_DIR, rootdir.toString()); this.ct = Mockito.mock(CatalogTracker.class); HRegionInterface hri = Mockito.mock(HRegionInterface.class); + Mockito.when(this.ct.getConnection()).thenReturn(this.connection); Mockito.when(ct.waitForMetaServerConnectionDefault()).thenReturn(hri); } @@ -115,9 +125,13 @@ public class TestCatalogJanitor { @Override public void stop(String why) { - //no-op + if (this.ct != null) { + this.ct.stop(); + } + if (this.connection != null) { + HConnectionManager.deleteConnection(this.connection.getConfiguration(), true); + } } - } /** @@ -267,42 +281,53 @@ public class TestCatalogJanitor { } @Test - public void testCleanParent() throws IOException { + public void testCleanParent() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); setRootDirAndCleanIt(htu, "testCleanParent"); Server server = new MockServer(htu); - MasterServices services = new MockMasterServices(server); - CatalogJanitor janitor = new CatalogJanitor(server, services); - // Create regions. - HTableDescriptor htd = createHTableDescriptor(); - HRegionInfo parent = - new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), - Bytes.toBytes("eee")); - HRegionInfo splita = - new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), - Bytes.toBytes("ccc")); - HRegionInfo splitb = - new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), - Bytes.toBytes("eee")); - // Test that when both daughter regions are in place, that we do not - // remove the parent. - Result r = createResult(parent, splita, splitb); - // Add a reference under splitA directory so we don't clear out the parent. - Path rootdir = services.getMasterFileSystem().getRootDir(); - Path tabledir = - HTableDescriptor.getTableDir(rootdir, htd.getName()); - Path storedir = Store.getStoreHomedir(tabledir, splita.getEncodedName(), - htd.getColumnFamilies()[0].getName()); - Reference ref = new Reference(Bytes.toBytes("ccc"), Reference.Range.top); - long now = System.currentTimeMillis(); - // Reference name has this format: StoreFile#REF_NAME_PARSER - Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); - FileSystem fs = services.getMasterFileSystem().getFileSystem(); - ref.write(fs, p); - assertFalse(janitor.cleanParent(parent, r)); - // Remove the reference file and try again. - assertTrue(fs.delete(p, true)); - assertTrue(janitor.cleanParent(parent, r)); + try { + MasterServices services = new MockMasterServices(server); + CatalogJanitor janitor = new CatalogJanitor(server, services); + // Create regions. + HTableDescriptor htd = new HTableDescriptor("table"); + htd.addFamily(new HColumnDescriptor("f")); + HRegionInfo parent = + new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("eee")); + HRegionInfo splita = + new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), + Bytes.toBytes("ccc")); + HRegionInfo splitb = + new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), + Bytes.toBytes("eee")); + // Test that when both daughter regions are in place, that we do not + // remove the parent. + List kvs = new ArrayList(); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITA_QUALIFIER, Writables.getBytes(splita))); + kvs.add(new KeyValue(parent.getRegionName(), HConstants.CATALOG_FAMILY, + HConstants.SPLITB_QUALIFIER, Writables.getBytes(splitb))); + Result r = new Result(kvs); + // Add a reference under splitA directory so we don't clear out the parent. + Path rootdir = services.getMasterFileSystem().getRootDir(); + Path tabledir = + HTableDescriptor.getTableDir(rootdir, htd.getName()); + Path storedir = Store.getStoreHomedir(tabledir, splita.getEncodedName(), + htd.getColumnFamilies()[0].getName()); + Reference ref = new Reference(Bytes.toBytes("ccc"), Reference.Range.top); + long now = System.currentTimeMillis(); + // Reference name has this format: StoreFile#REF_NAME_PARSER + Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); + FileSystem fs = services.getMasterFileSystem().getFileSystem(); + Path path = ref.write(fs, p); + assertTrue(fs.exists(path)); + assertFalse(janitor.cleanParent(parent, r)); + // Remove the reference file and try again. + assertTrue(fs.delete(p, true)); + assertTrue(janitor.cleanParent(parent, r)); + } finally { + server.stop("shutdown"); + } } /** @@ -459,4 +484,4 @@ public class TestCatalogJanitor { htd.addFamily(new HColumnDescriptor("f")); return htd; } -} \ No newline at end of file +} diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index a0e645011b2..ac7ead2da82 100644 --- a/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; @@ -98,6 +99,9 @@ public class TestDistributedLogSplitting { LOG.info("Waiting for active/ready master"); cluster.waitForActiveAndReadyMaster(); master = cluster.getMaster(); + while (cluster.getLiveRegionServerThreads().size() < num_rs) { + Threads.sleep(1); + } } @After @@ -111,7 +115,7 @@ public class TestDistributedLogSplitting { final int NUM_REGIONS_TO_CREATE = 40; final int NUM_ROWS_PER_REGION = 100; - startCluster(NUM_RS); + startCluster(NUM_RS); // NUM_RS=6. ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "distributed log splitting test", null); diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java index f473c807fe0..eaa47b97482 100644 --- a/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java @@ -86,21 +86,20 @@ public class TestMaster { tableRegions.get(0).getFirst().getEndKey()); // Now trigger a split and stop when the split is in progress - - CountDownLatch aboutToOpen = new CountDownLatch(1); + CountDownLatch split = new CountDownLatch(1); CountDownLatch proceed = new CountDownLatch(1); - RegionOpenListener list = new RegionOpenListener(aboutToOpen, proceed); + RegionSplitListener list = new RegionSplitListener(split, proceed); cluster.getMaster().executorService. - registerListener(EventType.RS_ZK_REGION_OPENED, list); + registerListener(EventType.RS_ZK_REGION_SPLIT, list); LOG.info("Splitting table"); admin.split(TABLENAME); LOG.info("Waiting for split result to be about to open"); - aboutToOpen.await(60, TimeUnit.SECONDS); + split.await(60, TimeUnit.SECONDS); try { LOG.info("Making sure we can call getTableRegions while opening"); - tableRegions = MetaReader.getTableRegionsAndLocations( - m.getCatalogTracker(), Bytes.toString(TABLENAME)); + tableRegions = MetaReader.getTableRegionsAndLocations(m.getCatalogTracker(), + TABLENAME, false); LOG.info("Regions: " + Joiner.on(',').join(tableRegions)); // We have three regions because one is split-in-progress @@ -118,22 +117,21 @@ public class TestMaster { } } - static class RegionOpenListener implements EventHandlerListener { - CountDownLatch aboutToOpen, proceed; + static class RegionSplitListener implements EventHandlerListener { + CountDownLatch split, proceed; - public RegionOpenListener(CountDownLatch aboutToOpen, CountDownLatch proceed) - { - this.aboutToOpen = aboutToOpen; + public RegionSplitListener(CountDownLatch split, CountDownLatch proceed) { + this.split = split; this.proceed = proceed; } @Override public void afterProcess(EventHandler event) { - if (event.getEventType() != EventType.RS_ZK_REGION_OPENED) { + if (event.getEventType() != EventType.RS_ZK_REGION_SPLIT) { return; } try { - aboutToOpen.countDown(); + split.countDown(); proceed.await(60, TimeUnit.SECONDS); } catch (InterruptedException ie) { throw new RuntimeException(ie); diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index bab2b8ad9b3..49bcf02c82e 100644 --- a/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -23,7 +23,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -34,11 +33,18 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.executor.EventHandler.EventType; +import org.apache.hadoop.hbase.executor.RegionTransitionData; import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; -import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.util.Bytes; @@ -49,8 +55,6 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKTable; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.NodeExistsException; import org.junit.Test; public class TestMasterFailover { @@ -131,7 +135,7 @@ public class TestMasterFailover { // Stop the cluster TEST_UTIL.shutdownMiniCluster(); } - + @Test public void testShouldCheckMasterFailOverWhenMETAIsInOpenedState() throws Exception { @@ -1006,4 +1010,4 @@ public class TestMasterFailover { private void log(String string) { LOG.info("\n\n" + string + " \n\n"); } -} +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java index 4de5e67b056..ffc8d9d699c 100644 --- a/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java +++ b/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java @@ -128,8 +128,10 @@ public class TestMergeTable { assertTrue("originalTableRegions=" + originalTableRegions.size() + ", postMergeTableRegions=" + postMergeTableRegions.size(), postMergeTableRegions.size() < originalTableRegions.size()); + LOG.info("Done with merge"); } finally { UTIL.shutdownMiniCluster(); + LOG.info("After cluster shutdown"); } } diff --git a/src/test/ruby/hbase/admin_test.rb b/src/test/ruby/hbase/admin_test.rb index 9a096e45321..df88100eee9 100644 --- a/src/test/ruby/hbase/admin_test.rb +++ b/src/test/ruby/hbase/admin_test.rb @@ -152,7 +152,7 @@ module Hbase assert_equal(['a:', 'b:'], table(@create_test_name).get_all_columns.sort) end - define_test "create hould work with hash column args" do + define_test "create should work with hash column args" do drop_test_table(@create_test_name) admin.create(@create_test_name, { NAME => 'a'}, { NAME => 'b'}) assert_equal(['a:', 'b:'], table(@create_test_name).get_all_columns.sort) @@ -160,14 +160,14 @@ module Hbase #------------------------------------------------------------------------------- -# define_test "close should work without region server name" do -# if admin.exists?(@create_test_name) -# admin.disable(@create_test_name) -# admin.drop(@create_test_name) -# end -# admin.create(@create_test_name, 'foo') -# admin.close_region(@create_test_name + ',,0') -# end + define_test "close should work without region server name" do + if admin.exists?(@create_test_name) + admin.disable(@create_test_name) + admin.drop(@create_test_name) + end + admin.create(@create_test_name, 'foo') + admin.close_region(@create_test_name + ',,0', nil) + end #------------------------------------------------------------------------------- @@ -187,13 +187,14 @@ module Hbase table(@test_name).put(1, "x:a", 1) table(@test_name).put(2, "x:a", 2) assert_equal(2, table(@test_name).count) - admin.truncate(@test_name) + # This is hacky. Need to get the configuration into admin instance + admin.truncate(@test_name, $TEST_CLUSTER.getConfiguration) assert_equal(0, table(@test_name).count) end define_test "truncate should yield log records" do logs = [] - admin.truncate(@test_name) do |log| + admin.truncate(@test_name, $TEST_CLUSTER.getConfiguration) do |log| assert_kind_of(String, log) logs << log end diff --git a/src/test/ruby/shell/shell_test.rb b/src/test/ruby/shell/shell_test.rb index 428958891a4..bc3000c30d1 100644 --- a/src/test/ruby/shell/shell_test.rb +++ b/src/test/ruby/shell/shell_test.rb @@ -25,7 +25,7 @@ require 'shell/formatter' class ShellTest < Test::Unit::TestCase def setup @formatter = ::Shell::Formatter::Console.new() - @hbase = ::Hbase::Hbase.new + @hbase = ::Hbase::Hbase.new($TEST_CLUSTER.getConfiguration) @shell = Shell::Shell.new(@hbase, @formatter) end