From 2fb19fb95102a7d2746f32c5a5da03eaae142b8f Mon Sep 17 00:00:00 2001 From: Enis Soztutar Date: Mon, 26 Jan 2015 10:56:59 -0800 Subject: [PATCH] HBASE-11908 Region replicas should be added to the meta table at the time of table creation --- .../hadoop/hbase/MetaTableAccessor.java | 54 +++++++++-- .../hadoop/hbase/master/RegionStateStore.java | 8 +- .../hadoop/hbase/master/RegionStates.java | 16 +++- .../hbase/master/TableStateManager.java | 4 + .../master/handler/CreateTableHandler.java | 6 +- .../master/handler/TruncateTableHandler.java | 2 +- .../master/snapshot/CloneSnapshotHandler.java | 5 +- .../snapshot/RestoreSnapshotHandler.java | 5 +- .../hadoop/hbase/TestMetaTableAccessor.java | 91 ++++++++++++++++++- .../hadoop/hbase/client/TestMetaScanner.java | 2 +- 10 files changed, 167 insertions(+), 26 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 108662ce57c..69cd543fbe9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -739,8 +739,14 @@ public class MetaTableAccessor { if (replicaId < 0) { break; } - - locations.add(getRegionLocation(r, regionInfo, replicaId)); + HRegionLocation location = getRegionLocation(r, regionInfo, replicaId); + // In case the region replica is newly created, it's location might be null. We usually do not + // have HRL's in RegionLocations object with null ServerName. They are handled as null HRLs. + if (location == null || location.getServerName() == null) { + locations.add(null); + } else { + locations.add(location); + } } return new RegionLocations(locations); @@ -1089,8 +1095,7 @@ public class MetaTableAccessor { * Adds a (single) hbase:meta row for the specified new region and its daughters. Note that this * does not add its daughter's as different rows, but adds information about the daughters * in the same row as the parent. Use - * {@link #splitRegion(org.apache.hadoop.hbase.client.Connection, - * HRegionInfo, HRegionInfo, HRegionInfo, ServerName)} + * {@link #splitRegion(Connection, HRegionInfo, HRegionInfo, HRegionInfo, ServerName, int) * if you want to do that. * @param meta the Table for META * @param regionInfo region information @@ -1112,7 +1117,7 @@ public class MetaTableAccessor { * Adds a (single) hbase:meta row for the specified new region and its daughters. Note that this * does not add its daughter's as different rows, but adds information about the daughters * in the same row as the parent. Use - * {@link #splitRegion(Connection, HRegionInfo, HRegionInfo, HRegionInfo, ServerName)} + * {@link #splitRegion(Connection, HRegionInfo, HRegionInfo, HRegionInfo, ServerName, int) * if you want to do that. * @param connection connection we're using * @param regionInfo region information @@ -1137,12 +1142,19 @@ public class MetaTableAccessor { * @throws IOException if problem connecting or updating meta */ public static void addRegionsToMeta(Connection connection, - List regionInfos) + List regionInfos, int regionReplication) throws IOException { List puts = new ArrayList(); for (HRegionInfo regionInfo : regionInfos) { if (RegionReplicaUtil.isDefaultReplica(regionInfo)) { puts.add(makePutFromRegionInfo(regionInfo)); + Put put = makePutFromRegionInfo(regionInfo); + // Add empty locations for region replicas so that number of replicas can be cached + // whenever the primary region is looked up from meta + for (int i = 1; i < regionReplication; i++) { + addEmptyLocation(put, i); + } + puts.add(put); } } putsToMetaTable(connection, puts); @@ -1180,7 +1192,8 @@ public class MetaTableAccessor { * @throws IOException */ public static void mergeRegions(final Connection connection, HRegionInfo mergedRegion, - HRegionInfo regionA, HRegionInfo regionB, ServerName sn) throws IOException { + HRegionInfo regionA, HRegionInfo regionB, ServerName sn, int regionReplication) + throws IOException { Table meta = getMetaHTable(connection); try { HRegionInfo copyOfMerged = new HRegionInfo(mergedRegion); @@ -1199,6 +1212,12 @@ public class MetaTableAccessor { // The merged is a new region, openSeqNum = 1 is fine. addLocation(putOfMerged, sn, 1, mergedRegion.getReplicaId()); + // Add empty locations for region replicas of the merged region so that number of replicas can + // be cached whenever the primary region is looked up from meta + for (int i = 1; i < regionReplication; i++) { + addEmptyLocation(putOfMerged, i); + } + byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString() + HConstants.DELIMITER); multiMutate(meta, tableRow, putOfMerged, deleteA, deleteB); @@ -1220,7 +1239,7 @@ public class MetaTableAccessor { */ public static void splitRegion(final Connection connection, HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB, - ServerName sn) throws IOException { + ServerName sn, int regionReplication) throws IOException { Table meta = getMetaHTable(connection); try { HRegionInfo copyOfParent = new HRegionInfo(parent); @@ -1238,6 +1257,13 @@ public class MetaTableAccessor { addLocation(putA, sn, 1, splitA.getReplicaId()); //new regions, openSeqNum = 1 is fine. addLocation(putB, sn, 1, splitB.getReplicaId()); + // Add empty locations for region replicas of daughters so that number of replicas can be + // cached whenever the primary region is looked up from meta + for (int i = 1; i < regionReplication; i++) { + addEmptyLocation(putA, i); + addEmptyLocation(putB, i); + } + byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER); multiMutate(meta, tableRow, putParent, putA, putB); } finally { @@ -1385,14 +1411,14 @@ public class MetaTableAccessor { * @throws IOException */ public static void overwriteRegions(Connection connection, - List regionInfos) throws IOException { + List regionInfos, int regionReplication) throws IOException { deleteRegions(connection, regionInfos); // Why sleep? This is the easiest way to ensure that the previous deletes does not // eclipse the following puts, that might happen in the same ts from the server. // See HBASE-9906, and HBASE-9879. Once either HBASE-9879, HBASE-8770 is fixed, // or HBASE-9905 is fixed and meta uses seqIds, we do not need the sleep. Threads.sleep(20); - addRegionsToMeta(connection, regionInfos); + addRegionsToMeta(connection, regionInfos, regionReplication); LOG.info("Overwritten " + regionInfos); } @@ -1433,4 +1459,12 @@ public class MetaTableAccessor { Bytes.toBytes(openSeqNum)); return p; } + + public static Put addEmptyLocation(final Put p, int replicaId) { + long now = EnvironmentEdgeManager.currentTime(); + p.addImmutable(HConstants.CATALOG_FAMILY, getServerColumn(replicaId), now, null); + p.addImmutable(HConstants.CATALOG_FAMILY, getStartCodeColumn(replicaId), now, null); + p.addImmutable(HConstants.CATALOG_FAMILY, getSeqNumColumn(replicaId), now, null); + return p; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java index 8f7d0f34519..313b03e3ab4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java @@ -239,12 +239,12 @@ public class RegionStateStore { } void splitRegion(HRegionInfo p, - HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException { - MetaTableAccessor.splitRegion(server.getConnection(), p, a, b, sn); + HRegionInfo a, HRegionInfo b, ServerName sn, int regionReplication) throws IOException { + MetaTableAccessor.splitRegion(server.getConnection(), p, a, b, sn, regionReplication); } void mergeRegions(HRegionInfo p, - HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException { - MetaTableAccessor.mergeRegions(server.getConnection(), p, a, b, sn); + HRegionInfo a, HRegionInfo b, ServerName sn, int regionReplication) throws IOException { + MetaTableAccessor.mergeRegions(server.getConnection(), p, a, b, sn, regionReplication); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index d4bd9a4624f..c98a998d44c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; @@ -772,7 +773,8 @@ public class RegionStates { void splitRegion(HRegionInfo p, HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException { - regionStateStore.splitRegion(p, a, b, sn); + + regionStateStore.splitRegion(p, a, b, sn, getRegionReplication(p)); synchronized (this) { // After PONR, split is considered to be done. // Update server holdings to be aligned with the meta. @@ -788,7 +790,7 @@ public class RegionStates { void mergeRegions(HRegionInfo p, HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException { - regionStateStore.mergeRegions(p, a, b, sn); + regionStateStore.mergeRegions(p, a, b, sn, getRegionReplication(a)); synchronized (this) { // After PONR, merge is considered to be done. // Update server holdings to be aligned with the meta. @@ -802,6 +804,16 @@ public class RegionStates { } } + private int getRegionReplication(HRegionInfo r) throws IOException { + if (tableStateManager != null) { + HTableDescriptor htd = tableStateManager.getTableDescriptors().get(r.getTable()); + if (htd != null) { + return htd.getRegionReplication(); + } + } + return 1; + } + /** * At cluster clean re/start, mark all user regions closed except those of tables * that are excluded, such as disabled/disabling/enabling tables. All user regions diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java index 04cc17c771b..d8199eaadf4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableStateManager.java @@ -183,6 +183,10 @@ public class TableStateManager { return tableState; } + TableDescriptors getTableDescriptors() { + return descriptors; + } + /** * Write descriptor in place, update cache of states. * Write lock should be hold by caller. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java index d1f01512b1c..2007ed4bf49 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java @@ -225,7 +225,7 @@ public class CreateTableHandler extends EventHandler { if (regionInfos != null && regionInfos.size() > 0) { // 4. Add regions to META - addRegionsToMeta(regionInfos); + addRegionsToMeta(regionInfos, hTableDescriptor.getRegionReplication()); // 5. Add replicas if needed regionInfos = addReplicas(hTableDescriptor, regionInfos); @@ -296,8 +296,8 @@ public class CreateTableHandler extends EventHandler { /** * Add the specified set of regions to the hbase:meta table. */ - protected void addRegionsToMeta(final List regionInfos) + protected void addRegionsToMeta(final List regionInfos, int regionReplication) throws IOException { - MetaTableAccessor.addRegionsToMeta(this.server.getConnection(), regionInfos); + MetaTableAccessor.addRegionsToMeta(this.server.getConnection(), regionInfos, regionReplication); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java index a124bf6ab57..ee40153da21 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TruncateTableHandler.java @@ -125,7 +125,7 @@ public class TruncateTableHandler extends DeleteTableHandler { // 4. Add regions to META MetaTableAccessor.addRegionsToMeta(masterServices.getConnection(), - regionInfos); + regionInfos, hTableDescriptor.getRegionReplication()); // 5. Trigger immediate assignment of the regions in round-robin fashion ModifyRegionUtils.assignRegions(assignmentManager, regionInfos); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java index c9fc93bb15a..2a6dca8951c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java @@ -138,9 +138,10 @@ public class CloneSnapshotHandler extends CreateTableHandler implements Snapshot } @Override - protected void addRegionsToMeta(final List regionInfos) + protected void addRegionsToMeta(final List regionInfos, + int regionReplication) throws IOException { - super.addRegionsToMeta(regionInfos); + super.addRegionsToMeta(regionInfos, regionReplication); metaChanges.updateMetaParentRegions(this.server.getConnection(), regionInfos); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java index 57895e96d16..56faf766d3c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java @@ -159,9 +159,10 @@ public class RestoreSnapshotHandler extends TableEventHandler implements Snapsho // in the snapshot folder. hris.clear(); if (metaChanges.hasRegionsToAdd()) hris.addAll(metaChanges.getRegionsToAdd()); - MetaTableAccessor.addRegionsToMeta(conn, hris); + MetaTableAccessor.addRegionsToMeta(conn, hris, hTableDescriptor.getRegionReplication()); if (metaChanges.hasRegionsToRestore()) { - MetaTableAccessor.overwriteRegions(conn, metaChanges.getRegionsToRestore()); + MetaTableAccessor.overwriteRegions(conn, metaChanges.getRegionsToRestore(), + hTableDescriptor.getRegionReplication()); } metaChanges.updateMetaParentRegions(this.server.getConnection(), hris); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java index 969394db3f9..e63797655f5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -49,6 +50,8 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import com.google.common.collect.Lists; + /** * Test {@link org.apache.hadoop.hbase.MetaTableAccessor}. */ @@ -58,6 +61,7 @@ public class TestMetaTableAccessor { private static final Log LOG = LogFactory.getLog(TestMetaTableAccessor.class); private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static Connection connection; + private Random random = new Random(); @BeforeClass public static void beforeClass() throws Exception { UTIL.startMiniCluster(3); @@ -320,7 +324,6 @@ public class TestMetaTableAccessor { @Test public void testMetaLocationsForRegionReplicas() throws IOException { - Random random = new Random(); ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong()); ServerName serverName1 = ServerName.valueOf("bar", 60010, random.nextLong()); ServerName serverName100 = ServerName.valueOf("baz", 60010, random.nextLong()); @@ -381,5 +384,91 @@ public class TestMetaTableAccessor { Bytes.toBytes(seqNum))); } } + + public static void assertEmptyMetaLocation(Table meta, byte[] row, int replicaId) + throws IOException { + Get get = new Get(row); + Result result = meta.get(get); + Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, + MetaTableAccessor.getServerColumn(replicaId)); + Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, + MetaTableAccessor.getStartCodeColumn(replicaId)); + assertNotNull(serverCell); + assertNotNull(startCodeCell); + assertEquals(0, serverCell.getValueLength()); + assertEquals(0, startCodeCell.getValueLength()); + } + + @Test + public void testMetaLocationForRegionReplicasIsAddedAtTableCreation() throws IOException { + long regionId = System.currentTimeMillis(); + HRegionInfo primary = new HRegionInfo(TableName.valueOf("table_foo"), + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 0); + + Table meta = MetaTableAccessor.getMetaHTable(connection); + try { + List regionInfos = Lists.newArrayList(primary); + MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3); + + assertEmptyMetaLocation(meta, primary.getRegionName(), 1); + assertEmptyMetaLocation(meta, primary.getRegionName(), 2); + } finally { + meta.close(); + } + } + + @Test + public void testMetaLocationForRegionReplicasIsAddedAtRegionSplit() throws IOException { + long regionId = System.currentTimeMillis(); + ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong()); + HRegionInfo parent = new HRegionInfo(TableName.valueOf("table_foo"), + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId, 0); + HRegionInfo splitA = new HRegionInfo(TableName.valueOf("table_foo"), + HConstants.EMPTY_START_ROW, Bytes.toBytes("a"), false, regionId+1, 0); + HRegionInfo splitB = new HRegionInfo(TableName.valueOf("table_foo"), + Bytes.toBytes("a"), HConstants.EMPTY_END_ROW, false, regionId+1, 0); + + + Table meta = MetaTableAccessor.getMetaHTable(connection); + try { + List regionInfos = Lists.newArrayList(parent); + MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3); + + MetaTableAccessor.splitRegion(connection, parent, splitA, splitB, serverName0, 3); + + assertEmptyMetaLocation(meta, splitA.getRegionName(), 1); + assertEmptyMetaLocation(meta, splitA.getRegionName(), 2); + assertEmptyMetaLocation(meta, splitB.getRegionName(), 1); + assertEmptyMetaLocation(meta, splitB.getRegionName(), 2); + } finally { + meta.close(); + } + } + + @Test + public void testMetaLocationForRegionReplicasIsAddedAtRegionMerge() throws IOException { + long regionId = System.currentTimeMillis(); + ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong()); + + HRegionInfo parentA = new HRegionInfo(TableName.valueOf("table_foo"), + Bytes.toBytes("a"), HConstants.EMPTY_END_ROW, false, regionId, 0); + HRegionInfo parentB = new HRegionInfo(TableName.valueOf("table_foo"), + HConstants.EMPTY_START_ROW, Bytes.toBytes("a"), false, regionId, 0); + HRegionInfo merged = new HRegionInfo(TableName.valueOf("table_foo"), + HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false, regionId+1, 0); + + Table meta = MetaTableAccessor.getMetaHTable(connection); + try { + List regionInfos = Lists.newArrayList(parentA, parentB); + MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3); + + MetaTableAccessor.mergeRegions(connection, merged, parentA, parentB, serverName0, 3); + + assertEmptyMetaLocation(meta, merged.getRegionName(), 1); + assertEmptyMetaLocation(meta, merged.getRegionName(), 2); + } finally { + meta.close(); + } + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java index eaab61b90d5..e195baf31ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaScanner.java @@ -166,7 +166,7 @@ public class TestMetaScanner { end); MetaTableAccessor.splitRegion(connection, - parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0)); + parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0), 1); Threads.sleep(random.nextInt(200)); } catch (Throwable e) {