From 53a7045cd9d0e77f5c3c2c310e8a712359737a62 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Tue, 9 Apr 2013 06:02:10 +0000 Subject: [PATCH] HBASE-8045 Fix .META. migration after HBASE-3171 git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1465894 13f79535-47bb-0310-9956-ffa450edef68 --- .../catalog/MetaMigrationConvertingToPB.java | 84 +++++-------------- .../apache/hadoop/hbase/master/HMaster.java | 8 +- .../TestMetaMigrationConvertingToPB.java | 15 ++-- 3 files changed, 31 insertions(+), 76 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationConvertingToPB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationConvertingToPB.java index 45764a5adcb..7d15d88867f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationConvertingToPB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationConvertingToPB.java @@ -34,8 +34,8 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.util.Bytes; /** - * A tool to migrate the data stored in ROOT and META tables to pbuf serialization. - * Supports migrating from 0.92.x and 0.94.x to 0.96.x for the catalog tables. + * A tool to migrate the data stored in META table to pbuf serialization. + * Supports migrating from 0.92.x and 0.94.x to 0.96.x for the catalog table. * @deprecated will be removed for the major release after 0.96. */ @Deprecated @@ -121,54 +121,28 @@ public class MetaMigrationConvertingToPB { } /** - * Update ROOT and META to newer version, converting writable serialization to PB, if - * it is needed. + * Converting writable serialization to PB, if it is needed. * @param services MasterServices to get a handle on master * @return num migrated rows * @throws IOException or RuntimeException if something goes wrong */ - public static long updateRootAndMetaIfNecessary(final MasterServices services) + public static long updateMetaIfNecessary(final MasterServices services) throws IOException { - if (isMetaHRIUpdated(services.getCatalogTracker())) { - LOG.info("ROOT/META already up-to date with PB serialization"); + if (isMetaTableUpdated(services.getCatalogTracker())) { + LOG.info("META already up-to date with PB serialization"); return 0; } - LOG.info("ROOT/META has Writable serializations, migrating ROOT and META to PB serialization"); + LOG.info("META has Writable serializations, migrating META to PB serialization"); try { - long rows = updateRootAndMeta(services); - LOG.info("ROOT and META updated with PB serialization. Total rows updated: " + rows); + long rows = updateMeta(services); + LOG.info("META updated with PB serialization. Total rows updated: " + rows); return rows; } catch (IOException e) { - LOG.warn("Update ROOT/META with PB serialization failed." + - "Master startup aborted."); + LOG.warn("Update META with PB serialization failed." + "Master startup aborted."); throw e; } } - /** - * Update ROOT and META to newer version, converting writable serialization to PB - * @return num migrated rows - */ - static long updateRootAndMeta(final MasterServices masterServices) - throws IOException { - long rows = updateRoot(masterServices); - rows += updateMeta(masterServices); - return rows; - } - - /** - * Update ROOT rows, converting writable serialization to PB - * @return num migrated rows - */ - static long updateRoot(final MasterServices masterServices) - throws IOException { - LOG.info("Starting update of ROOT"); - ConvertToPBMetaVisitor v = new ConvertToPBMetaVisitor(masterServices); - MetaReader.fullScan(masterServices.getCatalogTracker(), v, null); - LOG.info("Finished update of ROOT. Total rows updated:" + v.numMigratedRows); - return v.numMigratedRows; - } - /** * Update META rows, converting writable serialization to PB * @return num migrated rows @@ -177,45 +151,27 @@ public class MetaMigrationConvertingToPB { LOG.info("Starting update of META"); ConvertToPBMetaVisitor v = new ConvertToPBMetaVisitor(masterServices); MetaReader.fullScan(masterServices.getCatalogTracker(), v); - //updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker()); LOG.info("Finished update of META. Total rows updated:" + v.numMigratedRows); return v.numMigratedRows; } - /** - * Update the version flag in -ROOT-. - * @param catalogTracker the catalog tracker - * @throws IOException - */ - static void updateRootWithMetaMigrationStatus(final CatalogTracker catalogTracker) - throws IOException { - Put p = new Put(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); - p.add(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER, - Bytes.toBytes(HConstants.META_VERSION)); - // TODO so wrong - //MetaEditor.putToRootTable(catalogTracker, p); - LOG.info("Updated -ROOT- meta version=" + HConstants.META_VERSION); - } - /** * @param catalogTracker the catalog tracker * @return True if the meta table has been migrated. * @throws IOException */ - static boolean isMetaHRIUpdated(final CatalogTracker catalogTracker) throws IOException { + static boolean isMetaTableUpdated(final CatalogTracker catalogTracker) throws IOException { List results = MetaReader.fullScanOfMeta(catalogTracker); if (results == null || results.isEmpty()) { - LOG.info(".META. is not migrated"); - return false; + LOG.info(".META. doesn't have any entries to update."); + return true; } - // Presume only the one result because we only support one meta region. - Result r = results.get(0); - byte [] value = r.getValue(HConstants.CATALOG_FAMILY, - HConstants.META_VERSION_QUALIFIER); - short version = value == null || value.length <= 0? -1: Bytes.toShort(value); - - boolean migrated = version >= HConstants.META_VERSION; - LOG.info("Meta version=" + version + "; migrated=" + migrated); - return migrated; + for (Result r : results) { + byte[] value = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + if (!isMigrated(value)) { + return false; + } + } + return true; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 1a6dc50c336..ce8930b0deb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -764,12 +764,12 @@ Server { if (!assignMeta(status)) return; enableServerShutdownHandler(); - // Update meta with new PB serialization if required. i.e migrate all HRI - // to PB serialization in meta and update the status in ROOT. This must happen - // before we assign all user regions or else the assignment will fail. + // Update meta with new PB serialization if required. i.e migrate all HRI to PB serialization + // in meta. This must happen before we assign all user regions or else the assignment will + // fail. // TODO: Remove this after 0.96, when we do 0.98. org.apache.hadoop.hbase.catalog.MetaMigrationConvertingToPB - .updateRootAndMetaIfNecessary(this); + .updateMetaIfNecessary(this); this.balancer.setMasterServices(this); // Fix up assignment manager status diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java index 8066f7c8572..5083be1712f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java @@ -56,7 +56,6 @@ import org.junit.Test; import org.junit.experimental.categories.Category; /** - * TODO reenable the tests once a migration path is figured without ROOT * Test migration that changes HRI serialization into PB. Tests by bringing up a cluster from actual * data from a 0.92 cluster, as well as manually downgrading and then upgrading the META info. * @deprecated Remove after 0.96 @@ -169,16 +168,16 @@ public class TestMetaMigrationConvertingToPB { TEST_UTIL.shutdownMiniCluster(); } - //@Test + @Test public void testMetaUpdatedFlagInROOT() throws Exception { HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); boolean metaUpdated = MetaMigrationConvertingToPB. - isMetaHRIUpdated(master.getCatalogTracker()); + isMetaTableUpdated(master.getCatalogTracker()); assertEquals(true, metaUpdated); verifyMetaRowsAreUpdated(master.getCatalogTracker()); } - //@Test + @Test public void testMetaMigration() throws Exception { LOG.info("Starting testMetaMigration"); final byte [] FAMILY = Bytes.toBytes("family"); @@ -207,7 +206,7 @@ public class TestMetaMigrationConvertingToPB { // Assert that the flag in ROOT is updated to reflect the correct status boolean metaUpdated = - MetaMigrationConvertingToPB.isMetaHRIUpdated( + MetaMigrationConvertingToPB.isMetaTableUpdated( TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker()); assertEquals(true, metaUpdated); verifyMetaRowsAreUpdated(ct); @@ -225,7 +224,7 @@ public class TestMetaMigrationConvertingToPB { * rows and migrate any pending rows at startup. * @throws Exception */ - //@Test + @Test public void testMasterCrashDuringMetaMigration() throws Exception { final byte[] FAMILY = Bytes.toBytes("family"); HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration"); @@ -245,13 +244,13 @@ public class TestMetaMigrationConvertingToPB { LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI"); long numMigratedRows = - MetaMigrationConvertingToPB.updateRootAndMetaIfNecessary( + MetaMigrationConvertingToPB.updateMetaIfNecessary( TEST_UTIL.getHBaseCluster().getMaster()); assertEquals(numMigratedRows, 10); // Assert that the flag in ROOT is updated to reflect the correct status boolean metaUpdated = MetaMigrationConvertingToPB. - isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker()); + isMetaTableUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker()); assertEquals(true, metaUpdated); verifyMetaRowsAreUpdated(ct);