diff --git a/CHANGES.txt b/CHANGES.txt
index f68c823436f..9484e4f928b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -71,6 +71,7 @@ Release 0.19.0 - Unreleased
for root region"
HBASE-990 NoSuchElementException in flushSomeRegions
HBASE-602 HBase Crash when network card has a IPv6 address
+ HBASE-996 Migration script to up the versions in catalog tables
IMPROVEMENTS
HBASE-901 Add a limit to key length, check key and value length on client side
diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index dc7315f621d..319bc660795 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -23,7 +23,7 @@
# The java implementation to use. Required.
# export JAVA_HOME=/usr/lib/j2sdk1.5-sun
-export JAVA_HOME=/usr
+export JAVA_HOME=/Users/stack/bin/jdk
# Extra Java CLASSPATH elements. Optional.
# export HBASE_CLASSPATH=
diff --git a/src/java/org/apache/hadoop/hbase/HConstants.java b/src/java/org/apache/hadoop/hbase/HConstants.java
index de8bb422fa7..170fa0c9ef9 100644
--- a/src/java/org/apache/hadoop/hbase/HConstants.java
+++ b/src/java/org/apache/hadoop/hbase/HConstants.java
@@ -44,9 +44,10 @@ public interface HConstants {
/**
* Current version of file system
- * Version 4 supports only one kind of bloom filter
+ * Version 4 supports only one kind of bloom filter.
+ * Version 5 changes versions in catalog table regions.
*/
- public static final String FILE_SYSTEM_VERSION = "4";
+ public static final String FILE_SYSTEM_VERSION = "5";
// Configuration parameters
diff --git a/src/java/org/apache/hadoop/hbase/util/Migrate.java b/src/java/org/apache/hadoop/hbase/util/Migrate.java
index c849208166c..e07c40f2cce 100644
--- a/src/java/org/apache/hadoop/hbase/util/Migrate.java
+++ b/src/java/org/apache/hadoop/hbase/util/Migrate.java
@@ -21,40 +21,35 @@
package org.apache.hadoop.hbase.util;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.commons.cli.Options;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HStoreKey;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
- * Perform a file system upgrade to convert older file layouts.
+ * Perform a migration.
* HBase keeps a file in hdfs named hbase.version just under the hbase.rootdir.
* This file holds the version of the hbase data in the Filesystem. When the
* software changes in a manner incompatible with the data in the Filesystem,
* it updates its internal version number,
* {@link HConstants#FILE_SYSTEM_VERSION}. This wrapper script manages moving
* the filesystem across versions until there's a match with current software's
- * version number.
+ * version number. This script will only cross a particular version divide. You may
+ * need to install earlier or later hbase to migrate earlier (or older) versions.
*
* This wrapper script comprises a set of migration steps. Which steps
* are run depends on the span between the version of the hbase data in the
@@ -72,13 +67,12 @@ import org.apache.hadoop.util.ToolRunner;
* edits made by previous migration steps not being apparent in later migration
* steps). The upshot is always verify presumptions migrating.
*
- *
This script will migrate an hbase 0.1 install to a 0.2 install only.
+ *
This script will migrate an hbase 0.18.x only.
*
* @see How To Migration
*/
public class Migrate extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(Migrate.class);
-
private final HBaseConfiguration conf;
private FileSystem fs;
@@ -136,7 +130,7 @@ public class Migrate extends Configured implements Tool {
try {
// Verify file system is up.
fs = FileSystem.get(conf); // get DFS handle
- LOG.info("Verifying that file system is available...");
+ LOG.info("Verifying that file system is available..");
FSUtils.checkFileSystemAvailable(fs);
return true;
} catch (IOException e) {
@@ -147,7 +141,8 @@ public class Migrate extends Configured implements Tool {
private boolean notRunning() {
// Verify HBase is down
- LOG.info("Verifying that HBase is not running...");
+ LOG.info("Verifying that HBase is not running...." +
+ "Trys ten times to connect to running master");
try {
HBaseAdmin.checkHBaseAvailable(conf);
LOG.fatal("HBase cluster must be off-line.");
@@ -187,7 +182,7 @@ public class Migrate extends Configured implements Tool {
}
float version = Float.parseFloat(versionStr);
if (version == HBASE_0_1_VERSION ||
- Integer.valueOf(versionStr) < PREVIOUS_VERSION) {
+ Integer.valueOf(versionStr).intValue() < PREVIOUS_VERSION) {
String msg = "Cannot upgrade from " + versionStr + " to " +
HConstants.FILE_SYSTEM_VERSION + " you must install hbase-0.2.x, run " +
"the upgrade tool, reinstall this version and run this utility again." +
@@ -196,7 +191,7 @@ public class Migrate extends Configured implements Tool {
throw new IOException(msg);
}
- // insert call to new migration method here.
+ migrateTo5();
if (!readOnly) {
// Set file system version
@@ -213,37 +208,65 @@ public class Migrate extends Configured implements Tool {
}
}
- private FileStatus[] getRootDirFiles() throws IOException {
- FileStatus[] stats = fs.listStatus(FSUtils.getRootDir(this.conf));
- if (stats == null || stats.length == 0) {
- throw new IOException("No files found under root directory " +
- FSUtils.getRootDir(this.conf).toString());
+ // Move the fileystem version from 4 to 5.
+ // In here we rewrite the catalog table regions so they keep 10 versions
+ // instead of 1.
+ private void migrateTo5() throws IOException {
+ if (this.readOnly && this.migrationNeeded) {
+ return;
+ }
+ final MetaUtils utils = new MetaUtils(this.conf);
+ try {
+ updateVersions(utils.getRootRegion().getRegionInfo());
+ // Scan the root region
+ utils.scanRootRegion(new MetaUtils.ScannerListener() {
+ public boolean processRow(HRegionInfo info)
+ throws IOException {
+ if (readOnly && !migrationNeeded) {
+ migrationNeeded = true;
+ return false;
+ }
+ updateVersions(utils.getRootRegion(), info);
+ return true;
+ }
+ });
+ } finally {
+ utils.shutdown();
}
- return stats;
}
- private void checkForUnrecoveredLogFiles(FileStatus[] rootFiles)
+ /*
+ * Move from old pre-v5 hregioninfo to current HRegionInfo
+ * Persist back into r
+ * @param mr
+ * @param oldHri
+ */
+ void updateVersions(HRegion mr, HRegionInfo oldHri)
throws IOException {
- List unrecoveredLogs = new ArrayList();
- for (int i = 0; i < rootFiles.length; i++) {
- String name = rootFiles[i].getPath().getName();
- if (name.startsWith("log_")) {
- unrecoveredLogs.add(name);
- }
+ if (!updateVersions(oldHri)) {
+ return;
}
- if (unrecoveredLogs.size() != 0) {
- throw new IOException("There are " + unrecoveredLogs.size() +
- " unrecovered region server logs. Please uninstall this version of " +
- "HBase, re-install the previous version, start your cluster and " +
- "shut it down cleanly, so that all region server logs are recovered" +
- " and deleted. Or, if you are sure logs are vestige of old " +
- "failures in hbase, remove them and then rerun the migration. " +
- "See 'Redo Logs' in http://wiki.apache.org/hadoop/Hbase/HowToMigrate. " +
- "Here are the problem log files: " + unrecoveredLogs);
+ BatchUpdate b = new BatchUpdate(oldHri.getRegionName());
+ b.put(HConstants.COL_REGIONINFO, Writables.getBytes(oldHri));
+ mr.batchUpdate(b);
+ LOG.info("Upped versions on " + oldHri.getRegionNameAsString());
+ }
+
+ /*
+ * @param hri Update versions.
+ * @param true if we changed value
+ */
+ private boolean updateVersions(final HRegionInfo hri) {
+ HColumnDescriptor hcd =
+ hri.getTableDesc().getFamily(HConstants.COLUMN_FAMILY);
+ if (hcd.getMaxVersions() == 1) {
+ // Set it to 10, an arbitrary high number
+ hcd.setMaxVersions(10);
+ return true;
}
+ return false;
}
- @SuppressWarnings("static-access")
private int parseArgs(String[] args) {
Options opts = new Options();
GenericOptionsParser parser =
@@ -263,9 +286,11 @@ public class Migrate extends Configured implements Tool {
}
private void usage() {
- System.err.println("Usage: bin/hbase migrate { check | upgrade } [options]\n");
+ System.err.println("Usage: bin/hbase migrate {check | upgrade} [options]");
+ System.err.println();
System.err.println(" check perform upgrade checks only.");
- System.err.println(" upgrade perform upgrade checks and modify hbase.\n");
+ System.err.println(" upgrade perform upgrade checks and modify hbase.");
+ System.err.println();
System.err.println(" Options are:");
System.err.println(" -conf specify an application configuration file");
System.err.println(" -D use value for given property");
@@ -282,9 +307,9 @@ public class Migrate extends Configured implements Tool {
try {
status = ToolRunner.run(new Migrate(), args);
} catch (Exception e) {
- LOG.error("exiting due to error", e);
+ LOG.error(e);
status = -1;
}
System.exit(status);
}
-}
+}
\ No newline at end of file