From 8159b37b3e7bc4b9a4a8a6639cbaaa5bf8ca6bfa Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Thu, 16 Jun 2011 18:28:59 +0000 Subject: [PATCH] HBASE-3970 Address HMaster crash/failure half way through meta migration git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1136587 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 + .../hadoop/hbase/catalog/MetaEditor.java | 12 ++++- .../hadoop/hbase/HBaseTestingUtility.java | 32 +++++++++++++ .../hbase/client/TestMetaMigration.java | 48 +++++++++++++------ 4 files changed, 79 insertions(+), 15 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ebf38763d94..05835dde15c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -271,6 +271,8 @@ Release 0.91.0 - Unreleased HBASE-3573 Move shutdown messaging OFF hearbeat; prereq for fix of hbase-1502 HBASE-3071 Graceful decommissioning of a regionserver + HBASE-3970 Address HMaster crash/failure half way through meta migration + (Subbu M Iyer) NEW FEATURES HBASE-2001 Coprocessors: Colocate user code with regions (Mingjie Lai via diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java index eed7e0497fb..bf33f0c5ee6 100644 --- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java +++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java @@ -246,6 +246,7 @@ public class MetaEditor { public boolean visit(Result r) throws IOException { if (r == null || r.isEmpty()) return true; HRegionInfo090x hrfm = getHRegionInfoForMigration(r); + if (hrfm == null) return true; htds.add(hrfm.getTableDesc()); masterServices.getMasterFileSystem().createTableDescriptor(hrfm.getTableDesc()); HRegionInfo regionInfo = new HRegionInfo(hrfm); @@ -267,10 +268,19 @@ public class MetaEditor { public static HRegionInfo090x getHRegionInfoForMigration( Result data) throws IOException { + HRegionInfo090x info = null; byte [] bytes = data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); if (bytes == null) return null; - HRegionInfo090x info = Writables.getHRegionInfoForMigration(bytes); + try { + info = Writables.getHRegionInfoForMigration(bytes); + } catch(IOException ioe) { + if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) { + return null; + } else { + throw ioe; + } + } LOG.info("Current INFO from scan results = " + info); return info; } diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index ab89fa83ccb..6af0ecf5d53 100644 --- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -860,6 +860,22 @@ public class HBaseTestingUtility { return count; } + public int createMultiRegionsWithLegacyHRI(final Configuration c, + final HTableDescriptor htd, + final byte [] family, int numRegions) + throws IOException { + if (numRegions < 3) throw new IOException("Must create at least 3 regions"); + byte [] startKey = Bytes.toBytes("aaaaa"); + byte [] endKey = Bytes.toBytes("zzzzz"); + byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); + byte [][] regionStartKeys = new byte[splitKeys.length+1][]; + for (int i=0;i htds = MetaEditor.updateMetaWithNewRegionInfo( TEST_UTIL.getHBaseCluster().getMaster()); + MetaReader.fullScanMetaAndPrint(ct); assertEquals(3, htds.size()); // Assert that the flag in ROOT is updated to reflect the correct status boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated(); @@ -111,27 +112,46 @@ public class TestMetaMigration { } - //@Test - public void dtestUpdatesOnMetaWithNewHRI() throws Exception { - LOG.info("Starting testMetaWithLegacyHRI"); + /** + * This test assumes a master crash/failure during the meta migration process + * and attempts to continue the meta migration process when a new master takes over. + * When a master dies during the meta migration we will have some rows of + * META.CatalogFamily updated with new HRI, (i.e HRI with out HTD) and some + * still hanging with legacy HRI. (i.e HRI with HTD). When the backup master/ or + * fresh start of master attempts the migration it will encouter some rows of META + * already updated with new HRI and some still legacy. This test will simulate this + * scenario and validates that the migration process can safely skip the updated + * rows and migrate any pending rows at startup. + * @throws Exception + */ + @Test + public void testMasterCrashDuringMetaMigration() throws Exception { + LOG.info("Starting testMasterCrashDuringMetaMigration"); final byte[] FAMILY = Bytes.toBytes("family"); - HTableDescriptor htd = new HTableDescriptor("testMetaMigration"); + HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration"); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); htd.addFamily(hcd); Configuration conf = TEST_UTIL.getConfiguration(); - TEST_UTIL.createMultiRegionsWithNewHRI(conf, htd, FAMILY, - new byte[][]{ - HConstants.EMPTY_START_ROW, - Bytes.toBytes("region_a"), - Bytes.toBytes("region_b")}); + // Create 10 New regions. + TEST_UTIL.createMultiRegionsWithNewHRI(conf, htd, FAMILY, 10); + // Create 10 Legacy regions. + TEST_UTIL.createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10); + CatalogTracker ct = miniHBaseCluster.getMaster().getCatalogTracker(); + // just for this test set it to false. + MetaEditor.updateRootWithMetaMigrationStatus(ct, false); + //MetaReader.fullScanMetaAndPrint(ct); + LOG.info("MEta Print completed.testUpdatesOnMetaWithLegacyHRI"); + List htds = MetaEditor.updateMetaWithNewRegionInfo( TEST_UTIL.getHBaseCluster().getMaster()); - assertEquals(3, htds.size()); + assertEquals(10, htds.size()); + // Assert that the flag in ROOT is updated to reflect the correct status + boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated(); + assertEquals(true, metaUpdated); + LOG.info("END testMetaWithLegacyHRI"); + } - - - public static void assertEquals(int expected, int actual) { if (expected != actual) {