From 67fc6fdb020295cc3d79b8172e7a9513904d45aa Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 12 Nov 2008 20:44:06 +0000 Subject: [PATCH] HBASE-996 Migration script to up the versions in catalog tables git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@713502 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + conf/hbase-env.sh | 2 +- .../org/apache/hadoop/hbase/HConstants.java | 5 +- .../org/apache/hadoop/hbase/util/Migrate.java | 107 +++++++++++------- 4 files changed, 71 insertions(+), 44 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f68c823436f..9484e4f928b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -71,6 +71,7 @@ Release 0.19.0 - Unreleased for root region" HBASE-990 NoSuchElementException in flushSomeRegions HBASE-602 HBase Crash when network card has a IPv6 address + HBASE-996 Migration script to up the versions in catalog tables IMPROVEMENTS HBASE-901 Add a limit to key length, check key and value length on client side diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh index dc7315f621d..319bc660795 100644 --- a/conf/hbase-env.sh +++ b/conf/hbase-env.sh @@ -23,7 +23,7 @@ # The java implementation to use. Required. # export JAVA_HOME=/usr/lib/j2sdk1.5-sun -export JAVA_HOME=/usr +export JAVA_HOME=/Users/stack/bin/jdk # Extra Java CLASSPATH elements. Optional. # export HBASE_CLASSPATH= diff --git a/src/java/org/apache/hadoop/hbase/HConstants.java b/src/java/org/apache/hadoop/hbase/HConstants.java index de8bb422fa7..170fa0c9ef9 100644 --- a/src/java/org/apache/hadoop/hbase/HConstants.java +++ b/src/java/org/apache/hadoop/hbase/HConstants.java @@ -44,9 +44,10 @@ public interface HConstants { /** * Current version of file system - * Version 4 supports only one kind of bloom filter + * Version 4 supports only one kind of bloom filter. + * Version 5 changes versions in catalog table regions. */ - public static final String FILE_SYSTEM_VERSION = "4"; + public static final String FILE_SYSTEM_VERSION = "5"; // Configuration parameters diff --git a/src/java/org/apache/hadoop/hbase/util/Migrate.java b/src/java/org/apache/hadoop/hbase/util/Migrate.java index c849208166c..e07c40f2cce 100644 --- a/src/java/org/apache/hadoop/hbase/util/Migrate.java +++ b/src/java/org/apache/hadoop/hbase/util/Migrate.java @@ -21,40 +21,35 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import org.apache.commons.cli.Options; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HStoreKey; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.io.BatchUpdate; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; /** - * Perform a file system upgrade to convert older file layouts. + * Perform a migration. * HBase keeps a file in hdfs named hbase.version just under the hbase.rootdir. * This file holds the version of the hbase data in the Filesystem. When the * software changes in a manner incompatible with the data in the Filesystem, * it updates its internal version number, * {@link HConstants#FILE_SYSTEM_VERSION}. This wrapper script manages moving * the filesystem across versions until there's a match with current software's - * version number. + * version number. This script will only cross a particular version divide. You may + * need to install earlier or later hbase to migrate earlier (or older) versions. * *

This wrapper script comprises a set of migration steps. Which steps * are run depends on the span between the version of the hbase data in the @@ -72,13 +67,12 @@ import org.apache.hadoop.util.ToolRunner; * edits made by previous migration steps not being apparent in later migration * steps). The upshot is always verify presumptions migrating. * - *

This script will migrate an hbase 0.1 install to a 0.2 install only. + *

This script will migrate an hbase 0.18.x only. * * @see How To Migration */ public class Migrate extends Configured implements Tool { private static final Log LOG = LogFactory.getLog(Migrate.class); - private final HBaseConfiguration conf; private FileSystem fs; @@ -136,7 +130,7 @@ public class Migrate extends Configured implements Tool { try { // Verify file system is up. fs = FileSystem.get(conf); // get DFS handle - LOG.info("Verifying that file system is available..."); + LOG.info("Verifying that file system is available.."); FSUtils.checkFileSystemAvailable(fs); return true; } catch (IOException e) { @@ -147,7 +141,8 @@ public class Migrate extends Configured implements Tool { private boolean notRunning() { // Verify HBase is down - LOG.info("Verifying that HBase is not running..."); + LOG.info("Verifying that HBase is not running...." + + "Trys ten times to connect to running master"); try { HBaseAdmin.checkHBaseAvailable(conf); LOG.fatal("HBase cluster must be off-line."); @@ -187,7 +182,7 @@ public class Migrate extends Configured implements Tool { } float version = Float.parseFloat(versionStr); if (version == HBASE_0_1_VERSION || - Integer.valueOf(versionStr) < PREVIOUS_VERSION) { + Integer.valueOf(versionStr).intValue() < PREVIOUS_VERSION) { String msg = "Cannot upgrade from " + versionStr + " to " + HConstants.FILE_SYSTEM_VERSION + " you must install hbase-0.2.x, run " + "the upgrade tool, reinstall this version and run this utility again." + @@ -196,7 +191,7 @@ public class Migrate extends Configured implements Tool { throw new IOException(msg); } - // insert call to new migration method here. + migrateTo5(); if (!readOnly) { // Set file system version @@ -213,37 +208,65 @@ public class Migrate extends Configured implements Tool { } } - private FileStatus[] getRootDirFiles() throws IOException { - FileStatus[] stats = fs.listStatus(FSUtils.getRootDir(this.conf)); - if (stats == null || stats.length == 0) { - throw new IOException("No files found under root directory " + - FSUtils.getRootDir(this.conf).toString()); + // Move the fileystem version from 4 to 5. + // In here we rewrite the catalog table regions so they keep 10 versions + // instead of 1. + private void migrateTo5() throws IOException { + if (this.readOnly && this.migrationNeeded) { + return; + } + final MetaUtils utils = new MetaUtils(this.conf); + try { + updateVersions(utils.getRootRegion().getRegionInfo()); + // Scan the root region + utils.scanRootRegion(new MetaUtils.ScannerListener() { + public boolean processRow(HRegionInfo info) + throws IOException { + if (readOnly && !migrationNeeded) { + migrationNeeded = true; + return false; + } + updateVersions(utils.getRootRegion(), info); + return true; + } + }); + } finally { + utils.shutdown(); } - return stats; } - private void checkForUnrecoveredLogFiles(FileStatus[] rootFiles) + /* + * Move from old pre-v5 hregioninfo to current HRegionInfo + * Persist back into r + * @param mr + * @param oldHri + */ + void updateVersions(HRegion mr, HRegionInfo oldHri) throws IOException { - List unrecoveredLogs = new ArrayList(); - for (int i = 0; i < rootFiles.length; i++) { - String name = rootFiles[i].getPath().getName(); - if (name.startsWith("log_")) { - unrecoveredLogs.add(name); - } + if (!updateVersions(oldHri)) { + return; } - if (unrecoveredLogs.size() != 0) { - throw new IOException("There are " + unrecoveredLogs.size() + - " unrecovered region server logs. Please uninstall this version of " + - "HBase, re-install the previous version, start your cluster and " + - "shut it down cleanly, so that all region server logs are recovered" + - " and deleted. Or, if you are sure logs are vestige of old " + - "failures in hbase, remove them and then rerun the migration. " + - "See 'Redo Logs' in http://wiki.apache.org/hadoop/Hbase/HowToMigrate. " + - "Here are the problem log files: " + unrecoveredLogs); + BatchUpdate b = new BatchUpdate(oldHri.getRegionName()); + b.put(HConstants.COL_REGIONINFO, Writables.getBytes(oldHri)); + mr.batchUpdate(b); + LOG.info("Upped versions on " + oldHri.getRegionNameAsString()); + } + + /* + * @param hri Update versions. + * @param true if we changed value + */ + private boolean updateVersions(final HRegionInfo hri) { + HColumnDescriptor hcd = + hri.getTableDesc().getFamily(HConstants.COLUMN_FAMILY); + if (hcd.getMaxVersions() == 1) { + // Set it to 10, an arbitrary high number + hcd.setMaxVersions(10); + return true; } + return false; } - @SuppressWarnings("static-access") private int parseArgs(String[] args) { Options opts = new Options(); GenericOptionsParser parser = @@ -263,9 +286,11 @@ public class Migrate extends Configured implements Tool { } private void usage() { - System.err.println("Usage: bin/hbase migrate { check | upgrade } [options]\n"); + System.err.println("Usage: bin/hbase migrate {check | upgrade} [options]"); + System.err.println(); System.err.println(" check perform upgrade checks only."); - System.err.println(" upgrade perform upgrade checks and modify hbase.\n"); + System.err.println(" upgrade perform upgrade checks and modify hbase."); + System.err.println(); System.err.println(" Options are:"); System.err.println(" -conf specify an application configuration file"); System.err.println(" -D use value for given property"); @@ -282,9 +307,9 @@ public class Migrate extends Configured implements Tool { try { status = ToolRunner.run(new Migrate(), args); } catch (Exception e) { - LOG.error("exiting due to error", e); + LOG.error(e); status = -1; } System.exit(status); } -} +} \ No newline at end of file