Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-996 Migration script to up the versions in catalog tables 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@713502 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2008-11-12 20:44:06 +00:00
parent 01f09c1036
commit 67fc6fdb02
4 changed files with 71 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -71,6 +71,7 @@ Release 0.19.0 - Unreleased
for root region"
HBASE-990 NoSuchElementException in flushSomeRegions
HBASE-602 HBase Crash when network card has a IPv6 address
HBASE-996 Migration script to up the versions in catalog tables
IMPROVEMENTS
HBASE-901 Add a limit to key length, check key and value length on client side

2
conf/hbase-env.sh
View File

@ -23,7 +23,7 @@
# The java implementation to use. Required.
# export JAVA_HOME=/usr/lib/j2sdk1.5-sun
export JAVA_HOME=/usr
export JAVA_HOME=/Users/stack/bin/jdk
# Extra Java CLASSPATH elements. Optional.
# export HBASE_CLASSPATH=

5
src/java/org/apache/hadoop/hbase/HConstants.java
View File

@ -44,9 +44,10 @@ public interface HConstants {
/**
* Current version of file system
* Version 4 supports only one kind of bloom filter
* Version 4 supports only one kind of bloom filter.
* Version 5 changes versions in catalog table regions.
*/
public static final String FILE_SYSTEM_VERSION = "4";
public static final String FILE_SYSTEM_VERSION = "5";
// Configuration parameters

107
src/java/org/apache/hadoop/hbase/util/Migrate.java
View File

@ -21,40 +21,35 @@
package org.apache.hadoop.hbase.util;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.cli.Options;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* Perform a file system upgrade to convert older file layouts.
* Perform a migration.
* HBase keeps a file in hdfs named hbase.version just under the hbase.rootdir.
* This file holds the version of the hbase data in the Filesystem. When the
* software changes in a manner incompatible with the data in the Filesystem,
* it updates its internal version number,
* {@link HConstants#FILE_SYSTEM_VERSION}. This wrapper script manages moving
* the filesystem across versions until there's a match with current software's
* version number.
* version number. This script will only cross a particular version divide. You may
* need to install earlier or later hbase to migrate earlier (or older) versions.
*
* <p>This wrapper script comprises a set of migration steps. Which steps
* are run depends on the span between the version of the hbase data in the
@ -72,13 +67,12 @@ import org.apache.hadoop.util.ToolRunner;
* edits made by previous migration steps not being apparent in later migration
* steps). The upshot is always verify presumptions migrating.
*
* <p>This script will migrate an hbase 0.1 install to a 0.2 install only.
* <p>This script will migrate an hbase 0.18.x only.
*
* @see <a href="http://wiki.apache.org/hadoop/Hbase/HowToMigrate">How To Migration</a>
*/
public class Migrate extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(Migrate.class);
private final HBaseConfiguration conf;
private FileSystem fs;
@ -136,7 +130,7 @@ public class Migrate extends Configured implements Tool {
try {
// Verify file system is up.
fs = FileSystem.get(conf); // get DFS handle
LOG.info("Verifying that file system is available...");
LOG.info("Verifying that file system is available..");
FSUtils.checkFileSystemAvailable(fs);
return true;
} catch (IOException e) {
@ -147,7 +141,8 @@ public class Migrate extends Configured implements Tool {
private boolean notRunning() {
// Verify HBase is down
LOG.info("Verifying that HBase is not running...");
LOG.info("Verifying that HBase is not running...." +
"Trys ten times to connect to running master");
try {
HBaseAdmin.checkHBaseAvailable(conf);
LOG.fatal("HBase cluster must be off-line.");
@ -187,7 +182,7 @@ public class Migrate extends Configured implements Tool {
}
float version = Float.parseFloat(versionStr);
if (version == HBASE_0_1_VERSION ||
Integer.valueOf(versionStr) < PREVIOUS_VERSION) {
Integer.valueOf(versionStr).intValue() < PREVIOUS_VERSION) {
String msg = "Cannot upgrade from " + versionStr + " to " +
HConstants.FILE_SYSTEM_VERSION + " you must install hbase-0.2.x, run " +
"the upgrade tool, reinstall this version and run this utility again." +
@ -196,7 +191,7 @@ public class Migrate extends Configured implements Tool {
throw new IOException(msg);
}
// insert call to new migration method here.
migrateTo5();
if (!readOnly) {
// Set file system version
@ -213,37 +208,65 @@ public class Migrate extends Configured implements Tool {
}
}
private FileStatus[] getRootDirFiles() throws IOException {
FileStatus[] stats = fs.listStatus(FSUtils.getRootDir(this.conf));
if (stats == null || stats.length == 0) {
throw new IOException("No files found under root directory " +
FSUtils.getRootDir(this.conf).toString());
// Move the fileystem version from 4 to 5.
// In here we rewrite the catalog table regions so they keep 10 versions
// instead of 1.
private void migrateTo5() throws IOException {
if (this.readOnly && this.migrationNeeded) {
return;
}
final MetaUtils utils = new MetaUtils(this.conf);
try {
updateVersions(utils.getRootRegion().getRegionInfo());
// Scan the root region
utils.scanRootRegion(new MetaUtils.ScannerListener() {
public boolean processRow(HRegionInfo info)
throws IOException {
if (readOnly && !migrationNeeded) {
migrationNeeded = true;
return false;
}
updateVersions(utils.getRootRegion(), info);
return true;
}
});
} finally {
utils.shutdown();
}
return stats;
}
private void checkForUnrecoveredLogFiles(FileStatus[] rootFiles)
/*
* Move from old pre-v5 hregioninfo to current HRegionInfo
* Persist back into <code>r</code>
* @param mr
* @param oldHri
*/
void updateVersions(HRegion mr, HRegionInfo oldHri)
throws IOException {
List<String> unrecoveredLogs = new ArrayList<String>();
for (int i = 0; i < rootFiles.length; i++) {
String name = rootFiles[i].getPath().getName();
if (name.startsWith("log_")) {
unrecoveredLogs.add(name);
}
if (!updateVersions(oldHri)) {
return;
}
if (unrecoveredLogs.size() != 0) {
throw new IOException("There are " + unrecoveredLogs.size() +
" unrecovered region server logs. Please uninstall this version of " +
"HBase, re-install the previous version, start your cluster and " +
"shut it down cleanly, so that all region server logs are recovered" +
" and deleted. Or, if you are sure logs are vestige of old " +
"failures in hbase, remove them and then rerun the migration. " +
"See 'Redo Logs' in http://wiki.apache.org/hadoop/Hbase/HowToMigrate. " +
"Here are the problem log files: " + unrecoveredLogs);
BatchUpdate b = new BatchUpdate(oldHri.getRegionName());
b.put(HConstants.COL_REGIONINFO, Writables.getBytes(oldHri));
mr.batchUpdate(b);
LOG.info("Upped versions on " + oldHri.getRegionNameAsString());
}
/*
* @param hri Update versions.
* @param true if we changed value
*/
private boolean updateVersions(final HRegionInfo hri) {
HColumnDescriptor hcd =
hri.getTableDesc().getFamily(HConstants.COLUMN_FAMILY);
if (hcd.getMaxVersions() == 1) {
// Set it to 10, an arbitrary high number
hcd.setMaxVersions(10);
return true;
}
return false;
}
@SuppressWarnings("static-access")
private int parseArgs(String[] args) {
Options opts = new Options();
GenericOptionsParser parser =
@ -263,9 +286,11 @@ public class Migrate extends Configured implements Tool {
}
private void usage() {
System.err.println("Usage: bin/hbase migrate { check | upgrade } [options]\n");
System.err.println("Usage: bin/hbase migrate {check | upgrade} [options]");
System.err.println();
System.err.println(" check perform upgrade checks only.");
System.err.println(" upgrade perform upgrade checks and modify hbase.\n");
System.err.println(" upgrade perform upgrade checks and modify hbase.");
System.err.println();
System.err.println(" Options are:");
System.err.println(" -conf <configuration file> specify an application configuration file");
System.err.println(" -D <property=value> use value for given property");
@ -282,9 +307,9 @@ public class Migrate extends Configured implements Tool {
try {
status = ToolRunner.run(new Migrate(), args);
} catch (Exception e) {
LOG.error("exiting due to error", e);
LOG.error(e);
status = -1;
}
System.exit(status);
}
}
}