Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer) 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1167006 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Zhihong Yu 2011-09-09 05:25:24 +00:00
parent b043d60317
commit b723bc4d4d
8 changed files with 148 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -255,7 +255,7 @@ Release 0.91.0 - Unreleased
TestMultiColumnScanner that caught it (Mikhail Bautin)
HBASE-4007 distributed log splitting can get indefinitely stuck
(Prakash Khemani)
HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer)
IMPROVEMENTS

101
src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
View File

@ -231,27 +231,23 @@ public class MetaEditor {
}
/**
* Updates the region information for the specified region in META.
* Update the metamigrated flag in -ROOT-.
* @param catalogTracker
* @param regionInfo region to be updated in META
* @throws IOException
*/
public static void updateRegionInfo(CatalogTracker catalogTracker,
HRegionInfo regionInfo)
throws IOException {
Put put = new Put(regionInfo.getRegionName());
addRegionInfo(put, regionInfo);
catalogTracker.waitForMetaServerConnectionDefault().put(
CatalogTracker.META_REGION, put);
LOG.info("Updated region " + regionInfo.getRegionNameAsString() + " in META");
}
public static void updateRootWithMetaMigrationStatus(CatalogTracker catalogTracker) throws IOException {
public static void updateRootWithMetaMigrationStatus(
CatalogTracker catalogTracker) throws IOException {
updateRootWithMetaMigrationStatus(catalogTracker, true);
}
public static void updateRootWithMetaMigrationStatus(CatalogTracker catalogTracker,
boolean metaUpdated)
/**
* Update the metamigrated flag in -ROOT-.
* @param catalogTracker
* @param metaUpdated
* @throws IOException
*/
public static void updateRootWithMetaMigrationStatus(
CatalogTracker catalogTracker, boolean metaUpdated)
throws IOException {
Put put = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
addMetaUpdateStatus(put, metaUpdated);
@ -260,6 +256,12 @@ public class MetaEditor {
LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
}
/**
* Update legacy META rows, removing HTD from HRI.
* @param masterServices
* @return
* @throws IOException
*/
public static List<HTableDescriptor> updateMetaWithNewRegionInfo(
final MasterServices masterServices)
throws IOException {
@ -271,16 +273,11 @@ public class MetaEditor {
HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
if (hrfm == null) return true;
htds.add(hrfm.getTableDesc());
masterServices.getMasterFileSystem().createTableDescriptor(hrfm.getTableDesc());
HRegionInfo regionInfo = new HRegionInfo(hrfm);
LOG.debug(" MetaEditor.updatemeta RegionInfo = " + regionInfo.toString()
+ " old HRI = " + hrfm.toString());
Put put = new Put(regionInfo.getRegionName());
put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(regionInfo));
masterServices.getCatalogTracker().waitForMetaServerConnectionDefault().put(
CatalogTracker.META_REGION, put);
LOG.info("Updated region " + regionInfo + " to META");
masterServices.getMasterFileSystem()
.createTableDescriptor(hrfm.getTableDesc());
updateHRI(masterServices.getCatalogTracker()
.waitForMetaServerConnectionDefault(),
hrfm, CatalogTracker.META_REGION);
return true;
}
};
@ -289,6 +286,60 @@ public class MetaEditor {
return htds;
}
/**
* Migrate root and meta to newer version. This updates the META and ROOT
* and removes the HTD from HRI.
* @param masterServices
* @throws IOException
*/
public static void migrateRootAndMeta(final MasterServices masterServices)
throws IOException {
updateRootWithNewRegionInfo(masterServices);
updateMetaWithNewRegionInfo(masterServices);
}
/**
* Update the ROOT with new HRI. (HRI with no HTD)
* @param masterServices
* @return
* @throws IOException
*/
public static List<HTableDescriptor> updateRootWithNewRegionInfo(
final MasterServices masterServices)
throws IOException {
final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
Visitor v = new Visitor() {
@Override
public boolean visit(Result r) throws IOException {
if (r == null || r.isEmpty()) return true;
HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
if (hrfm == null) return true;
htds.add(hrfm.getTableDesc());
masterServices.getMasterFileSystem().createTableDescriptor(
hrfm.getTableDesc());
updateHRI(masterServices.getCatalogTracker()
.waitForRootServerConnectionDefault(),
hrfm, CatalogTracker.ROOT_REGION);
return true;
}
};
MetaReader.fullScan(
masterServices.getCatalogTracker().waitForRootServerConnectionDefault(),
v, HRegionInfo.ROOT_REGIONINFO.getRegionName(), null);
return htds;
}
private static void updateHRI(HRegionInterface hRegionInterface,
HRegionInfo090x hRegionInfo090x, byte[] regionName)
throws IOException {
HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x);
Put put = new Put(regionInfo.getRegionName());
put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(regionInfo));
hRegionInterface.put(regionName, put);
LOG.info("Updated region " + regionInfo + " to " + Bytes.toString(regionName));
}
public static HRegionInfo090x getHRegionInfoForMigration(
Result data) throws IOException {
HRegionInfo090x info = null;

42
src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
View File

@ -252,19 +252,8 @@ public class MetaReader {
throws IOException {
HRegionInterface metaServer =
catalogTracker.waitForMetaServerConnectionDefault();
Scan scan = new Scan();
if (startrow != null) scan.setStartRow(startrow);
scan.addFamily(HConstants.CATALOG_FAMILY);
long scannerid = metaServer.openScanner(
HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
try {
Result data;
while((data = metaServer.next(scannerid)) != null) {
if (!data.isEmpty()) visitor.visit(data);
}
} finally {
metaServer.close(scannerid);
}
fullScan(metaServer, visitor,
HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), startrow);
return;
}
@ -622,7 +611,32 @@ public class MetaReader {
return regions;
}
/**
* Fully scan a given region, on a given server starting with given row.
* @param hRegionInterface region server
* @param visitor visitor
* @param regionName name of region
* @param startrow start row
* @throws IOException
*/
public static void fullScan(HRegionInterface hRegionInterface,
Visitor visitor, final byte[] regionName,
byte[] startrow) throws IOException {
if (hRegionInterface == null) return;
Scan scan = new Scan();
if (startrow != null) scan.setStartRow(startrow);
scan.addFamily(HConstants.CATALOG_FAMILY);
long scannerid = hRegionInterface.openScanner(regionName, scan);
try {
Result data;
while((data = hRegionInterface.next(scannerid)) != null) {
if (!data.isEmpty()) visitor.visit(data);
}
} finally {
hRegionInterface.close(scannerid);
}
return;
}
/**

3
src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
View File

@ -914,7 +914,8 @@ public class HConnectionManager {
// possible we got a region of a different table...
if (!Bytes.equals(regionInfo.getTableName(), tableName)) {
throw new TableNotFoundException(
"Table '" + Bytes.toString(tableName) + "' was not found.");
"Table '" + Bytes.toString(tableName) + "' was not found, got: " +
Bytes.toString(regionInfo.getTableName()) + ".");
}
if (regionInfo.isSplit()) {
throw new RegionOfflineException("the only available region for" +

10
src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -526,14 +526,15 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
if (!isMetaHRIUpdated()) {
LOG.info("Meta has HRI with HTDs. Updating meta now.");
try {
MetaEditor.updateMetaWithNewRegionInfo(this);
LOG.info("Meta updated with new HRI.");
MetaEditor.migrateRootAndMeta(this);
LOG.info("ROOT and Meta updated with new HRI.");
return true;
} catch (IOException e) {
throw new RuntimeException("Update Meta with nw HRI failed. Master startup aborted.");
throw new RuntimeException("Update ROOT/Meta with new HRI failed." +
"Master startup aborted.");
}
}
LOG.info("Meta already up-to date with new HRI.");
LOG.info("ROOT/Meta already up-to date with new HRI.");
return true;
}
@ -977,7 +978,6 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
return hRegionInfos;
}
private static boolean isCatalogTable(final byte [] tableName) {
return Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) ||
Bytes.equals(tableName, HConstants.META_TABLE_NAME);

9
src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
View File

@ -327,9 +327,18 @@ public class MasterFileSystem {
if (!FSUtils.rootRegionExists(fs, rd)) {
bootstrap(rd, c);
}
createRootTableInfo(rd);
return rd;
}
private void createRootTableInfo(Path rd) throws IOException {
// Create ROOT tableInfo if required.
if (!FSUtils.tableInfoExists(fs, rd,
Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) {
FSUtils.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC, this.conf);
}
}
private static void bootstrap(final Path rd, final Configuration c)
throws IOException {
LOG.info("BOOTSTRAP: creating ROOT and first META regions");

14
src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
View File

@ -31,6 +31,9 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* Implementation of {@link TableDescriptors} that reads descriptors from the
@ -39,6 +42,8 @@ import org.apache.hadoop.hbase.TableExistsException;
* the filesystem or can be read and write.
*/
public class FSTableDescriptors implements TableDescriptors {
private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class);
private final FileSystem fs;
private final Path rootdir;
private final boolean fsreadonly;
@ -151,7 +156,14 @@ public class FSTableDescriptors implements TableDescriptors {
Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
List<Path> tableDirs = FSUtils.getTableDirs(fs, rootdir);
for (Path d: tableDirs) {
HTableDescriptor htd = get(d.getName());
HTableDescriptor htd = null;
try {
htd = get(d.getName());
} catch (FileNotFoundException fnfe) {
// inability of retrieving one HTD shouldn't stop getting the remaining
LOG.warn("Trouble retrieving htd", fnfe);
}
if (htd == null) continue;
htds.put(d.getName(), htd);
}

14
src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
View File

@ -490,6 +490,20 @@ public abstract class FSUtils {
return fs.exists(rootRegionDir);
}
/**
* Checks if .tableinfo exists for given table
*
* @param fs file system
* @param rootdir root directory of HBase installation
* @param tableName name of table
* @return true if exists
* @throws IOException
*/
public static boolean tableInfoExists(FileSystem fs, Path rootdir,
String tableName) throws IOException {
Path tablePath = getTableInfoPath(rootdir, tableName);
return fs.exists(tablePath);
}
/**
* Compute HDFS blocks distribution of a given file, or a portion of the file