From f05f116327c1f0dc8479ce238b3a47454dfea13c Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Tue, 21 Aug 2018 21:38:01 +0900 Subject: [PATCH] HBASE-21084 When cloning a snapshot including a split parent region, the split parent region of the cloned table will be online --- .../procedure/CloneSnapshotProcedure.java | 23 +++++- .../client/TestCloneSnapshotFromClient.java | 72 +++++++++++++++++-- 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java index 12bda2d6f4a..9ba34857473 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java @@ -39,6 +39,8 @@ import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.master.MetricsSnapshot; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; @@ -149,7 +151,26 @@ public class CloneSnapshotProcedure break; case CLONE_SNAPSHOT_ASSIGN_REGIONS: CreateTableProcedure.setEnablingState(env, getTableName()); - addChildProcedure(env.getAssignmentManager().createRoundRobinAssignProcedures(newRegions)); + + // Separate newRegions to split regions and regions to assign + List splitRegions = new ArrayList<>(); + List regionsToAssign = new ArrayList<>(); + newRegions.forEach(ri -> { + if (ri.isOffline() && (ri.isSplit() || ri.isSplitParent())) { + splitRegions.add(ri); + } else { + regionsToAssign.add(ri); + } + }); + + // For split regions, add them to RegionStates + AssignmentManager am = env.getAssignmentManager(); + splitRegions.forEach(ri -> + am.getRegionStates().updateRegionState(ri, RegionState.State.SPLIT) + ); + + addChildProcedure(env.getAssignmentManager() + .createRoundRobinAssignProcedures(regionsToAssign)); setNextState(CloneSnapshotState.CLONE_SNAPSHOT_UPDATE_DESC_CACHE); break; case CLONE_SNAPSHOT_UPDATE_DESC_CACHE: diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java index f3dd8077459..157b18f2d26 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java @@ -17,13 +17,19 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.Assert.assertEquals; + import java.io.IOException; +import java.util.List; + import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; @@ -160,7 +166,8 @@ public class TestCloneSnapshotFromClient { @Test(expected=SnapshotDoesNotExistException.class) public void testCloneNonExistentSnapshot() throws IOException, InterruptedException { String snapshotName = "random-snapshot-" + System.currentTimeMillis(); - final TableName tableName = TableName.valueOf(name.getMethodName() + "-" + System.currentTimeMillis()); + final TableName tableName = TableName.valueOf(name.getMethodName() + "-" + + System.currentTimeMillis()); admin.cloneSnapshot(snapshotName, tableName); } @@ -172,7 +179,8 @@ public class TestCloneSnapshotFromClient { @Test public void testCloneSnapshot() throws IOException, InterruptedException { - final TableName clonedTableName = TableName.valueOf(name.getMethodName() + "-" + System.currentTimeMillis()); + final TableName clonedTableName = TableName.valueOf(name.getMethodName() + "-" + + System.currentTimeMillis()); testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows); testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows); testCloneSnapshot(clonedTableName, emptySnapshot, 0); @@ -196,7 +204,8 @@ public class TestCloneSnapshotFromClient { public void testCloneSnapshotCrossNamespace() throws IOException, InterruptedException { String nsName = "testCloneSnapshotCrossNamespace"; admin.createNamespace(NamespaceDescriptor.create(nsName).build()); - final TableName clonedTableName = TableName.valueOf(nsName, name.getMethodName() + "-" + System.currentTimeMillis()); + final TableName clonedTableName = TableName.valueOf(nsName, name.getMethodName() + + "-" + System.currentTimeMillis()); testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows); testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows); testCloneSnapshot(clonedTableName, emptySnapshot, 0); @@ -208,7 +217,8 @@ public class TestCloneSnapshotFromClient { @Test public void testCloneLinksAfterDelete() throws IOException, InterruptedException { // Clone a table from the first snapshot - final TableName clonedTableName = TableName.valueOf(name.getMethodName() + "1-" + System.currentTimeMillis()); + final TableName clonedTableName = TableName.valueOf(name.getMethodName() + "1-" + + System.currentTimeMillis()); admin.cloneSnapshot(snapshotName0, clonedTableName); verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows); @@ -217,7 +227,8 @@ public class TestCloneSnapshotFromClient { admin.snapshot(snapshotName2, clonedTableName); // Clone the snapshot of the cloned table - final TableName clonedTableName2 = TableName.valueOf(name.getMethodName() + "2-" + System.currentTimeMillis()); + final TableName clonedTableName2 = TableName.valueOf(name.getMethodName() + "2-" + + System.currentTimeMillis()); admin.cloneSnapshot(snapshotName2, clonedTableName2); verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows); admin.disableTable(clonedTableName2); @@ -244,7 +255,8 @@ public class TestCloneSnapshotFromClient { verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows); // Clone a new table from cloned - final TableName clonedTableName3 = TableName.valueOf(name.getMethodName() + "3-" + System.currentTimeMillis()); + final TableName clonedTableName3 = TableName.valueOf(name.getMethodName() + "3-" + + System.currentTimeMillis()); admin.cloneSnapshot(snapshotName2, clonedTableName3); verifyRowCount(TEST_UTIL, clonedTableName3, snapshot0Rows); @@ -254,6 +266,49 @@ public class TestCloneSnapshotFromClient { admin.deleteSnapshot(snapshotName2); } + @Test + public void testCloneSnapshotAfterSplittingRegion() throws IOException, InterruptedException { + // Turn off the CatalogJanitor + admin.catalogJanitorSwitch(false); + + try { + List regionInfos = admin.getRegions(tableName); + RegionReplicaUtil.removeNonDefaultRegions(regionInfos); + + // Split the first region + splitRegion(regionInfos.get(0)); + + // Take a snapshot + admin.snapshot(snapshotName2, tableName); + + // Clone the snapshot to another table + TableName clonedTableName = TableName.valueOf(name.getMethodName() + "-" + + System.currentTimeMillis()); + admin.cloneSnapshot(snapshotName2, clonedTableName); + SnapshotTestingUtils.waitForTableToBeOnline(TEST_UTIL, clonedTableName); + + RegionStates regionStates = + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); + + // The region count of the cloned table should be the same as the one of the original table + int openRegionCountOfOriginalTable = + regionStates.getRegionByStateOfTable(tableName).get(RegionState.State.OPEN).size(); + int openRegionCountOfClonedTable = + regionStates.getRegionByStateOfTable(clonedTableName).get(RegionState.State.OPEN).size(); + assertEquals(openRegionCountOfOriginalTable, openRegionCountOfClonedTable); + + int splitRegionCountOfOriginalTable = + regionStates.getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT).size(); + int splitRegionCountOfClonedTable = + regionStates.getRegionByStateOfTable(clonedTableName).get(RegionState.State.SPLIT).size(); + assertEquals(splitRegionCountOfOriginalTable, splitRegionCountOfClonedTable); + + TEST_UTIL.deleteTable(clonedTableName); + } finally { + admin.catalogJanitorSwitch(true); + } + } + // ========================================================================== // Helpers // ========================================================================== @@ -266,4 +321,9 @@ public class TestCloneSnapshotFromClient { long expectedRows) throws IOException { SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows); } + + protected void splitRegion(final RegionInfo regionInfo) throws IOException { + byte[][] splitPoints = Bytes.split(regionInfo.getStartKey(), regionInfo.getEndKey(), 1); + admin.split(regionInfo.getTable(), splitPoints[1]); + } }