HBASE-2943 major_compact (and other admin commands) broken for .META.

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@992110 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2010-09-02 20:39:41 +00:00
parent 1574bac2f1
commit e3d0498438
10 changed files with 310 additions and 28 deletions

View File

@ -500,6 +500,7 @@ Release 0.21.0 - Unreleased
append by config or by missing lib/feature
HBASE-2799 "Append not enabled" warning should not show if hbase
root dir isn't on DFS
HBASE-2943 major_compact (and other admin commands) broken for .META.
IMPROVEMENTS
HBASE-1760 Cleanup TODOs in HTable

View File

@ -312,6 +312,10 @@ public class AvroServer {
try {
admin.flush(Bytes.toBytes(table));
return null;
} catch (InterruptedException e) {
AIOError ioe = new AIOError();
ioe.message = new Utf8(e.getMessage());
throw ioe;
} catch (IOException e) {
AIOError ioe = new AIOError();
ioe.message = new Utf8(e.getMessage());
@ -324,6 +328,10 @@ public class AvroServer {
try {
admin.split(Bytes.toBytes(table));
return null;
} catch (InterruptedException e) {
AIOError ioe = new AIOError();
ioe.message = new Utf8(e.getMessage());
throw ioe;
} catch (IOException e) {
AIOError ioe = new AIOError();
ioe.message = new Utf8(e.getMessage());

View File

@ -48,6 +48,7 @@ import org.apache.zookeeper.KeeperException;
* be explicitly set. Instead, ZooKeeper is used to learn of the availability
* and location of ROOT. ROOT is used to learn of the location of META. If not
* available in ROOT, ZooKeeper is used to monitor for a new location of META.
* <p>Call {@link #start()} to start up operation.
*/
public class CatalogTracker {
private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
@ -79,6 +80,21 @@ public class CatalogTracker {
* @param abortable if fatal exception
* @throws IOException
*/
public CatalogTracker(final ZooKeeperWatcher zk,
final ServerConnection connection, final Abortable abortable)
throws IOException {
this(zk, connection, abortable, 0);
}
/**
* Constructs the catalog tracker. Find current state of catalog tables and
* begin active tracking by executing {@link #start()}.
* @param zk
* @param connection server connection
* @param abortable if fatal exception
* @param defaultTimeout Timeout to use.
* @throws IOException
*/
public CatalogTracker(final ZooKeeperWatcher zk,
final ServerConnection connection, final Abortable abortable,
final int defaultTimeout)
@ -285,7 +301,7 @@ public class CatalogTracker {
}
if(getMetaServerConnection(true) == null) {
throw new NotAllMetaRegionsOnlineException(
"Timed out (" + timeout + "ms");
"Timed out (" + timeout + "ms)");
}
return metaLocation;
}

View File

@ -24,14 +24,14 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.NavigableSet;
import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
@ -47,6 +47,73 @@ import org.apache.hadoop.hbase.util.Writables;
* catalogs.
*/
public class MetaReader {
public static final byte [] META_REGION_PREFIX;
static {
// Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
// FIRST_META_REGIONINFO == '.META.,,1'. META_REGION_PREFIX == '.META.,'
int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
META_REGION_PREFIX = new byte [len];
System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
META_REGION_PREFIX, 0, len);
}
/**
* @param ct
* @param tableName A user tablename or a .META. table name.
* @return Interface on to server hosting the <code>-ROOT-</code> or
* <code>.META.</code> regions.
* @throws NotAllMetaRegionsOnlineException
* @throws IOException
*/
private static HRegionInterface getCatalogRegionInterface(final CatalogTracker ct,
final byte [] tableName)
throws NotAllMetaRegionsOnlineException, IOException {
return Bytes.equals(HConstants.META_TABLE_NAME, tableName)?
ct.waitForRootServerConnectionDefault():
ct.waitForMetaServerConnectionDefault();
}
/**
* @param tableName
* @return Returns region name to look in for regions for <code>tableName</code>;
* e.g. if we are looking for <code>.META.</code> regions, we need to look
* in the <code>-ROOT-</code> region, else if a user table, we need to look
* in the <code>.META.</code> region.
*/
private static byte [] getCatalogRegionNameForTable(final byte [] tableName) {
return Bytes.equals(HConstants.META_TABLE_NAME, tableName)?
HRegionInfo.ROOT_REGIONINFO.getRegionName():
HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
}
/**
* @param regionName
* @return Returns region name to look in for <code>regionName</code>;
* e.g. if we are looking for <code>.META.,,1</code> region, we need to look
* in <code>-ROOT-</code> region, else if a user region, we need to look
* in the <code>.META.,,1</code> region.
*/
private static byte [] getCatalogRegionNameForRegion(final byte [] regionName) {
return isMetaRegion(regionName)?
HRegionInfo.ROOT_REGIONINFO.getRegionName():
HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
}
/**
* @param regionName
* @return True if <code>regionName</code> is from <code>.META.</code> table.
*/
private static boolean isMetaRegion(final byte [] regionName) {
if (regionName.length < META_REGION_PREFIX.length + 2 /* ',', + '1' */) {
// Can't be meta table region.
return false;
}
// Compare the prefix of regionName. If it matches META_REGION_PREFIX prefix,
// then this is region from .META. table.
return Bytes.compareTo(regionName, 0, META_REGION_PREFIX.length,
META_REGION_PREFIX, 0, META_REGION_PREFIX.length) == 0;
}
/**
* Performs a full scan of <code>.META.</code>.
* <p>
@ -57,8 +124,7 @@ public class MetaReader {
* @return map of regions to their currently assigned server
* @throws IOException
*/
public static Map<HRegionInfo,HServerAddress> fullScan(
CatalogTracker catalogTracker)
public static Map<HRegionInfo,HServerAddress> fullScan(CatalogTracker catalogTracker)
throws IOException {
HRegionInterface metaServer =
catalogTracker.waitForMetaServerConnectionDefault();
@ -105,6 +171,7 @@ public class MetaReader {
public static HServerAddress readRegionLocation(CatalogTracker catalogTracker,
byte [] regionName)
throws IOException {
if (isMetaRegion(regionName)) throw new IllegalArgumentException("See readMetaLocation");
return readLocation(catalogTracker.waitForMetaServerConnectionDefault(),
CatalogTracker.META_REGION, regionName);
}
@ -155,14 +222,19 @@ public class MetaReader {
throws IOException {
Get get = new Get(regionName);
get.addFamily(HConstants.CATALOG_FAMILY);
Result r = catalogTracker.waitForMetaServerConnectionDefault().get(
CatalogTracker.META_REGION, get);
byte [] meta = getCatalogRegionNameForRegion(regionName);
Result r = catalogTracker.waitForMetaServerConnectionDefault().get(meta, get);
if(r == null || r.isEmpty()) {
return null;
}
return metaRowToRegionPair(r);
}
/**
* @param data A .META. table row.
* @return A pair of the regioninfo and the server address from <code>data</code>.
* @throws IOException
*/
public static Pair<HRegionInfo, HServerAddress> metaRowToRegionPair(
Result data) throws IOException {
HRegionInfo info = Writables.getHRegionInfo(
@ -188,6 +260,11 @@ public class MetaReader {
public static boolean tableExists(CatalogTracker catalogTracker,
String tableName)
throws IOException {
if (tableName.equals(HTableDescriptor.ROOT_TABLEDESC.getNameAsString()) ||
tableName.equals(HTableDescriptor.META_TABLEDESC.getNameAsString())) {
// Catalog tables always exist.
return true;
}
HRegionInterface metaServer =
catalogTracker.waitForMetaServerConnectionDefault();
byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
@ -213,24 +290,37 @@ public class MetaReader {
}
/**
* Gets all of the regions of the specified table from META.
* Gets all of the regions of the specified table.
* @param catalogTracker
* @param tableName
* @return
* @return Ordered list of {@link HRegionInfo}.
* @throws IOException
*/
public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
byte [] tableName)
throws IOException {
if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
// If root, do a bit of special handling.
List<HRegionInfo> list = new ArrayList<HRegionInfo>();
list.add(HRegionInfo.ROOT_REGIONINFO);
return list;
} else if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
// Same for .META. table
List<HRegionInfo> list = new ArrayList<HRegionInfo>();
list.add(HRegionInfo.FIRST_META_REGIONINFO);
return list;
}
// Its a user table.
HRegionInterface metaServer =
catalogTracker.waitForMetaServerConnectionDefault();
getCatalogRegionInterface(catalogTracker, tableName);
List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
String tableString = Bytes.toString(tableName);
byte[] firstRowInTable = Bytes.toBytes(tableString + ",,");
Scan scan = new Scan(firstRowInTable);
scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
long scannerid = metaServer.openScanner(
HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
long scannerid =
metaServer.openScanner(getCatalogRegionNameForTable(tableName), scan);
try {
Result data;
while((data = metaServer.next(scannerid)) != null) {
@ -251,18 +341,34 @@ public class MetaReader {
}
}
/**
* @param catalogTracker
* @param tableName
* @return Return list of regioninfos and server addresses.
* @throws IOException
* @throws InterruptedException
*/
public static List<Pair<HRegionInfo, HServerAddress>>
getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName)
throws IOException {
throws IOException, InterruptedException {
byte [] tableNameBytes = Bytes.toBytes(tableName);
if (Bytes.equals(tableNameBytes, HConstants.ROOT_TABLE_NAME)) {
// If root, do a bit of special handling.
HServerAddress hsa = catalogTracker.getRootLocation();
List<Pair<HRegionInfo, HServerAddress>> list =
new ArrayList<Pair<HRegionInfo, HServerAddress>>();
list.add(new Pair<HRegionInfo, HServerAddress>(HRegionInfo.ROOT_REGIONINFO, hsa));
return list;
}
HRegionInterface metaServer =
catalogTracker.waitForMetaServerConnectionDefault();
getCatalogRegionInterface(catalogTracker, tableNameBytes);
List<Pair<HRegionInfo, HServerAddress>> regions =
new ArrayList<Pair<HRegionInfo, HServerAddress>>();
byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
Scan scan = new Scan(firstRowInTable);
scan.addFamily(HConstants.CATALOG_FAMILY);
long scannerid = metaServer.openScanner(
HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
long scannerid =
metaServer.openScanner(getCatalogRegionNameForTable(tableNameBytes), scan);
try {
Result data;
while((data = metaServer.next(scannerid)) != null) {
@ -282,8 +388,15 @@ public class MetaReader {
}
}
/**
* @param catalogTracker
* @param hsi Server specification
* @return List of user regions installed on this server (does not include
* catalog regions).
* @throws IOException
*/
public static NavigableMap<HRegionInfo, Result>
getServerRegions(CatalogTracker catalogTracker, final HServerInfo hsi)
getServerUserRegions(CatalogTracker catalogTracker, final HServerInfo hsi)
throws IOException {
HRegionInterface metaServer =
catalogTracker.waitForMetaServerConnectionDefault();

View File

@ -740,8 +740,10 @@ public class HBaseAdmin implements Abortable {
*
* @param tableNameOrRegionName table or region to flush
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
public void flush(final String tableNameOrRegionName) throws IOException {
public void flush(final String tableNameOrRegionName)
throws IOException, InterruptedException {
flush(Bytes.toBytes(tableNameOrRegionName));
}
@ -751,8 +753,10 @@ public class HBaseAdmin implements Abortable {
*
* @param tableNameOrRegionName table or region to flush
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
public void flush(final byte [] tableNameOrRegionName) throws IOException {
public void flush(final byte [] tableNameOrRegionName)
throws IOException, InterruptedException {
boolean isRegionName = isRegionName(tableNameOrRegionName);
if (isRegionName) {
Pair<HRegionInfo, HServerAddress> pair =
@ -780,8 +784,10 @@ public class HBaseAdmin implements Abortable {
*
* @param tableNameOrRegionName table or region to compact
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
public void compact(final String tableNameOrRegionName) throws IOException {
public void compact(final String tableNameOrRegionName)
throws IOException, InterruptedException {
compact(Bytes.toBytes(tableNameOrRegionName));
}
@ -791,8 +797,10 @@ public class HBaseAdmin implements Abortable {
*
* @param tableNameOrRegionName table or region to compact
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
public void compact(final byte [] tableNameOrRegionName) throws IOException {
public void compact(final byte [] tableNameOrRegionName)
throws IOException, InterruptedException {
compact(tableNameOrRegionName, false);
}
@ -802,9 +810,10 @@ public class HBaseAdmin implements Abortable {
*
* @param tableNameOrRegionName table or region to major compact
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
public void majorCompact(final String tableNameOrRegionName)
throws IOException {
throws IOException, InterruptedException {
majorCompact(Bytes.toBytes(tableNameOrRegionName));
}
@ -814,9 +823,10 @@ public class HBaseAdmin implements Abortable {
*
* @param tableNameOrRegionName table or region to major compact
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
public void majorCompact(final byte [] tableNameOrRegionName)
throws IOException {
throws IOException, InterruptedException {
compact(tableNameOrRegionName, true);
}
@ -827,9 +837,10 @@ public class HBaseAdmin implements Abortable {
* @param tableNameOrRegionName table or region to compact
* @param major True if we are to do a major compaction.
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
private void compact(final byte [] tableNameOrRegionName, final boolean major)
throws IOException {
throws IOException, InterruptedException {
if (isRegionName(tableNameOrRegionName)) {
Pair<HRegionInfo, HServerAddress> pair =
MetaReader.getRegion(getCatalogTracker(), tableNameOrRegionName);
@ -882,8 +893,10 @@ public class HBaseAdmin implements Abortable {
*
* @param tableNameOrRegionName table or region to split
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
public void split(final String tableNameOrRegionName) throws IOException {
public void split(final String tableNameOrRegionName)
throws IOException, InterruptedException {
split(Bytes.toBytes(tableNameOrRegionName));
}
@ -893,8 +906,9 @@ public class HBaseAdmin implements Abortable {
*
* @param tableNameOrRegionName table to region to split
* @throws IOException if a remote or network exception occurs
* @throws InterruptedException
*/
public void split(final byte [] tableNameOrRegionName) throws IOException {
public void split(final byte [] tableNameOrRegionName) throws IOException, InterruptedException {
if (isRegionName(tableNameOrRegionName)) {
// Its a possible region name.
Pair<HRegionInfo, HServerAddress> pair =

View File

@ -108,7 +108,7 @@ public interface HConnection {
* lives in.
* @param tableName name of the table <i>row</i> is in
* @param row row key you're trying to find the region of
* @return HRegionLocation that describes where to find the reigon in
* @return HRegionLocation that describes where to find the region in
* question
* @throws IOException if a remote or network exception occurs
*/

View File

@ -104,7 +104,7 @@ public class ServerShutdownHandler extends EventHandler {
}
NavigableMap<HRegionInfo, Result> hris =
MetaReader.getServerRegions(this.server.getCatalogTracker(), this.hsi);
MetaReader.getServerUserRegions(this.server.getCatalogTracker(), this.hsi);
LOG.info("Reassigning the " + hris.size() + " region(s) that " + serverName +
" was carrying.");

View File

@ -1883,6 +1883,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
@Override
public void flushRegion(HRegionInfo regionInfo)
throws NotServingRegionException, IOException {
LOG.info("Flushing " + regionInfo.getRegionNameAsString());
HRegion region = getRegion(regionInfo.getRegionName());
region.flushcache();
}

View File

@ -222,6 +222,8 @@ public class ThriftServer {
public void compact(byte[] tableNameOrRegionName) throws IOError {
try{
admin.compact(tableNameOrRegionName);
} catch (InterruptedException e) {
throw new IOError(e.getMessage());
} catch (IOException e) {
throw new IOError(e.getMessage());
}
@ -230,6 +232,8 @@ public class ThriftServer {
public void majorCompact(byte[] tableNameOrRegionName) throws IOError {
try{
admin.majorCompact(tableNameOrRegionName);
} catch (InterruptedException e) {
throw new IOError(e.getMessage());
} catch (IOException e) {
throw new IOError(e.getMessage());
}

View File

@ -0,0 +1,125 @@
/**
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.catalog;
import static org.junit.Assert.*;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.ServerConnection;
import org.apache.hadoop.hbase.client.ServerConnectionManager;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* Test {@link MetaReader}, {@link MetaEditor}, and {@link RootLocationEditor}.
*/
public class TestMetaReaderEditor {
private static final Log LOG = LogFactory.getLog(TestMetaReaderEditor.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static ZooKeeperWatcher ZKW;
private static CatalogTracker CT;
private final static Abortable ABORTABLE = new Abortable() {
private final AtomicBoolean abort = new AtomicBoolean(false);
@Override
public void abort(String why, Throwable e) {
LOG.info(why, e);
abort.set(true);
}
};
@BeforeClass public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
ZKW = new ZooKeeperWatcher(UTIL.getConfiguration(),
"TestMetaReaderEditor", ABORTABLE);
ServerConnection connection =
ServerConnectionManager.getConnection(UTIL.getConfiguration());
CT = new CatalogTracker(ZKW, connection, ABORTABLE);
CT.start();
}
@AfterClass public static void afterClass() throws IOException {
UTIL.shutdownMiniCluster();
}
@Test public void testGetRegionsCatalogTables()
throws IOException, InterruptedException {
List<HRegionInfo> regions =
MetaReader.getTableRegions(CT, HConstants.META_TABLE_NAME);
assertTrue(regions.size() >= 1);
assertTrue(MetaReader.getTableRegionsAndLocations(CT,
Bytes.toString(HConstants.META_TABLE_NAME)).size() >= 1);
assertTrue(MetaReader.getTableRegionsAndLocations(CT,
Bytes.toString(HConstants.ROOT_TABLE_NAME)).size() == 1);
}
@Test public void testTableExists() throws IOException {
final String name = "testTableExists";
final byte [] nameBytes = Bytes.toBytes(name);
assertFalse(MetaReader.tableExists(CT, name));
UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
assertTrue(MetaReader.tableExists(CT, name));
UTIL.getHBaseAdmin().disableTable(name);
UTIL.getHBaseAdmin().deleteTable(name);
assertFalse(MetaReader.tableExists(CT, name));
assertTrue(MetaReader.tableExists(CT,
Bytes.toString(HConstants.META_TABLE_NAME)));
assertTrue(MetaReader.tableExists(CT,
Bytes.toString(HConstants.ROOT_TABLE_NAME)));
}
@Test public void testGetRegion() throws IOException, InterruptedException {
final String name = "testGetRegion";
final byte [] nameBytes = Bytes.toBytes(name);
HTable t = UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY);
// Test it works getting a region from user table.
List<HRegionInfo> regions = MetaReader.getTableRegions(CT, nameBytes);
assertEquals(regionCount, regions.size());
Pair<HRegionInfo, HServerAddress> pair =
MetaReader.getRegion(CT, regions.get(0).getRegionName());
assertEquals(regions.get(0).getEncodedName(),
pair.getFirst().getEncodedName());
// Test get on non-existent region.
pair = MetaReader.getRegion(CT, Bytes.toBytes("nonexistent-region"));
assertNull(pair);
// Test it works getting a region from meta/root.
pair =
MetaReader.getRegion(CT, HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
assertEquals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(),
pair.getFirst().getEncodedName());
}
}