HBASE-10672. Table snapshot should handle tables whose REGION_REPLICATION is greater than one

git-svn-id: https://svn.apache.org/repos/asf/hbase/branches/hbase-10070@1575096 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Devaraj Das 2014-03-06 23:33:23 +00:00 committed by Enis Soztutar
parent aed0216810
commit b0480db1ae
14 changed files with 238 additions and 38 deletions

View File

@ -18,6 +18,9 @@
package org.apache.hadoop.hbase.client;
import java.util.Collection;
import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hbase.HRegionInfo;
@ -71,4 +74,18 @@ public class RegionReplicaUtil {
public static boolean isDefaultReplica(HRegionInfo hri) {
return hri.getReplicaId() == DEFAULT_REPLICA_ID;
}
/**
* Removes the non-default replicas from the passed regions collection
* @param regions
*/
public static void removeNonDefaultRegions(Collection<HRegionInfo> regions) {
Iterator<HRegionInfo> iterator = regions.iterator();
while (iterator.hasNext()) {
HRegionInfo hri = iterator.next();
if (!RegionReplicaUtil.isDefaultReplica(hri)) {
iterator.remove();
}
}
}
}

View File

@ -1299,31 +1299,21 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
private HRegionInfo[] getHRegionInfos(HTableDescriptor hTableDescriptor,
byte[][] splitKeys) {
HRegionInfo[] hRegionInfos = null;
int numRegionReplicas = hTableDescriptor.getRegionReplication();
if (numRegionReplicas <= 0) {
LOG.warn("Invalid number of replicas per region in the table descriptor. Setting it to 1.");
numRegionReplicas = 1;
}
long regionId = System.currentTimeMillis();
HRegionInfo[] hRegionInfos = null;
if (splitKeys == null || splitKeys.length == 0) {
hRegionInfos = new HRegionInfo[numRegionReplicas];
for (int i = 0; i < numRegionReplicas; i++) {
hRegionInfos[i] = new HRegionInfo(hTableDescriptor.getTableName(), null, null,
false, regionId, (short)i);
}
hRegionInfos = new HRegionInfo[]{new HRegionInfo(hTableDescriptor.getTableName(), null, null,
false, regionId)};
} else {
int numRegions = splitKeys.length + 1;
hRegionInfos = new HRegionInfo[numRegions * numRegionReplicas];
hRegionInfos = new HRegionInfo[numRegions];
byte[] startKey = null;
byte[] endKey = null;
for (int i = 0; i < numRegions; i++) {
endKey = (i == splitKeys.length) ? null : splitKeys[i];
for (int j = 0; j < numRegionReplicas; j++) {
hRegionInfos[i*numRegionReplicas + j] =
new HRegionInfo(hTableDescriptor.getTableName(), startKey, endKey,
false, regionId, (short)j);
}
hRegionInfos[i] =
new HRegionInfo(hTableDescriptor.getTableName(), startKey, endKey,
false, regionId);
startKey = endKey;
}
}

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.handler;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.executor.EventType;
import org.apache.hadoop.hbase.master.AssignmentManager;
@ -84,6 +86,7 @@ public class CreateTableHandler extends EventHandler {
, EventType.C_M_CREATE_TABLE.toString());
}
@Override
public CreateTableHandler prepare()
throws NotAllMetaRegionsOnlineException, TableExistsException, IOException {
int timeout = conf.getInt("hbase.client.catalog.timeout", 10000);
@ -240,8 +243,10 @@ public class CreateTableHandler extends EventHandler {
if (regionInfos != null && regionInfos.size() > 0) {
// 4. Add regions to META
addRegionsToMeta(this.catalogTracker, regionInfos);
// 5. Add replicas if needed
regionInfos = addReplicas(hTableDescriptor, regionInfos);
// 5. Trigger immediate assignment of the regions in round-robin fashion
// 6. Trigger immediate assignment of the regions in round-robin fashion
ModifyRegionUtils.assignRegions(assignmentManager, regionInfos);
}
@ -255,6 +260,30 @@ public class CreateTableHandler extends EventHandler {
}
}
/**
* Create any replicas for the regions (the default replicas that was
* already created is passed to the method)
* @param hTableDescriptor
* @param regions default replicas
* @return the combined list of default and non-default replicas
*/
protected List<HRegionInfo> addReplicas(HTableDescriptor hTableDescriptor,
List<HRegionInfo> regions) {
int numRegionReplicas = hTableDescriptor.getRegionReplication() - 1;
if (numRegionReplicas <= 0) {
return regions;
}
List<HRegionInfo> hRegionInfos =
new ArrayList<HRegionInfo>((numRegionReplicas+1)*regions.size());
for (int i = 0; i < regions.size(); i++) {
for (int j = 1; j <= numRegionReplicas; j++) {
hRegionInfos.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(i), j));
}
}
hRegionInfos.addAll(regions);
return hRegionInfos;
}
private void releaseTableLock() {
if (this.tableLock != null) {
try {

View File

@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
@ -84,7 +85,11 @@ public class DisabledTableSnapshotHandler extends TakeSnapshotHandler {
// extract each pair to separate lists
Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
for (Pair<HRegionInfo, ServerName> p : regionsAndLocations) {
regions.add(p.getFirst());
// Don't include non-default regions
HRegionInfo hri = p.getFirst();
if (RegionReplicaUtil.isDefaultReplica(hri)) {
regions.add(hri);
}
}
// 2. for each region, write all the info to disk

View File

@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
@ -151,6 +152,8 @@ public final class MasterSnapshotVerifier {
private void verifyRegions(final SnapshotManifest manifest) throws IOException {
List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
tableName);
// Remove the non-default regions
RegionReplicaUtil.removeNonDefaultRegions(regions);
Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
if (regionManifests == null) {

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver.snapshot;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@ -36,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DaemonThreadFactory;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.master.snapshot.MasterSnapshotVerifier;
@ -220,7 +222,16 @@ public class RegionServerSnapshotManager extends RegionServerProcedureManager {
* @throws IOException
*/
private List<HRegion> getRegionsToSnapshot(SnapshotDescription snapshot) throws IOException {
return rss.getOnlineRegions(TableName.valueOf(snapshot.getTable()));
List<HRegion> onlineRegions = rss.getOnlineRegions(TableName.valueOf(snapshot.getTable()));
Iterator<HRegion> iterator = onlineRegions.iterator();
// remove the non-default regions
while (iterator.hasNext()) {
HRegion r = iterator.next();
if (!RegionReplicaUtil.isDefaultReplica(r.getRegionInfo())) {
iterator.remove();
}
}
return onlineRegions;
}
/**

View File

@ -143,13 +143,7 @@ public abstract class ModifyRegionUtils {
CompletionService<HRegionInfo> completionService =
new ExecutorCompletionService<HRegionInfo>(exec);
List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
int defaultReplicas = 0;
for (final HRegionInfo newRegion : newRegions) {
regionInfos.add(newRegion);
if (!RegionReplicaUtil.isDefaultReplica(newRegion)) {
continue;
}
defaultReplicas++;
completionService.submit(new Callable<HRegionInfo>() {
@Override
public HRegionInfo call() throws IOException {
@ -159,8 +153,8 @@ public abstract class ModifyRegionUtils {
}
try {
// wait for all regions to finish creation
for (int i = 0; i < defaultReplicas; i++) {
completionService.take().get();
for (int i = 0; i < regionNumber; i++) {
regionInfos.add(completionService.take().get());
}
} catch (InterruptedException e) {
LOG.error("Caught " + e + " during region creation");

View File

@ -99,7 +99,7 @@ public class TestCloneSnapshotFromClient {
snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
// create Table and disable it
SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
admin.disableTable(tableName);
// take an empty snapshot
@ -132,6 +132,10 @@ public class TestCloneSnapshotFromClient {
}
}
protected int getNumReplicas() {
return 1;
}
@After
public void tearDown() throws Exception {
if (admin.tableExists(tableName)) {
@ -168,9 +172,14 @@ public class TestCloneSnapshotFromClient {
admin.cloneSnapshot(snapshotName, tableName);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshotRows);
verifyReplicasCameOnline(tableName);
TEST_UTIL.deleteTable(tableName);
}
protected void verifyReplicasCameOnline(TableName tableName) throws IOException {
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
}
@Test
public void testCloneSnapshotCrossNamespace() throws IOException, InterruptedException {
String nsName = "testCloneSnapshotCrossNamespace";

View File

@ -0,0 +1,30 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.client;
import org.apache.hadoop.hbase.LargeTests;
import org.junit.experimental.categories.Category;
@Category(LargeTests.class)
public class TestCloneSnapshotFromClientWithRegionReplicas extends
TestCloneSnapshotFromClient {
@Override
protected int getNumReplicas() {
return 3;
}
}

View File

@ -104,7 +104,7 @@ public class TestRestoreSnapshotFromClient {
snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
// create Table and disable it
SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
admin.disableTable(tableName);
// take an empty snapshot
@ -143,23 +143,31 @@ public class TestRestoreSnapshotFromClient {
admin.restoreSnapshot(snapshotName0);
admin.enableTable(tableName);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
// Restore from emptySnapshot
admin.disableTable(tableName);
admin.restoreSnapshot(emptySnapshot);
admin.enableTable(tableName);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
// Restore from snapshot-1
admin.disableTable(tableName);
admin.restoreSnapshot(snapshotName1);
admin.enableTable(tableName);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
// Restore from snapshot-1
TEST_UTIL.deleteTable(tableName);
admin.restoreSnapshot(snapshotName1);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
}
protected int getNumReplicas() {
return 1;
}
@Test
@ -224,6 +232,7 @@ public class TestRestoreSnapshotFromClient {
TableName.valueOf("clonedtb-" + System.currentTimeMillis());
admin.cloneSnapshot(snapshotName0, clonedTableName);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
admin.disableTable(clonedTableName);
admin.snapshot(snapshotName2, clonedTableName);
TEST_UTIL.deleteTable(clonedTableName);
@ -231,6 +240,7 @@ public class TestRestoreSnapshotFromClient {
admin.cloneSnapshot(snapshotName2, clonedTableName);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
TEST_UTIL.deleteTable(clonedTableName);
}
@ -241,12 +251,14 @@ public class TestRestoreSnapshotFromClient {
admin.cloneSnapshot(snapshotName0, tableName);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
waitCleanerRun();
admin.disableTable(tableName);
admin.restoreSnapshot(snapshotName0);
admin.enableTable(tableName);
SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
}
@Test

View File

@ -0,0 +1,30 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.client;
import org.apache.hadoop.hbase.LargeTests;
import org.junit.experimental.categories.Category;
@Category(LargeTests.class)
public class TestRestoreSnapshotFromClientWithRegionReplicas extends
TestRestoreSnapshotFromClient {
@Override
protected int getNumReplicas() {
return 3;
}
}

View File

@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@ -57,11 +58,11 @@ import com.google.common.collect.Lists;
@Category(LargeTests.class)
public class TestSnapshotFromClient {
private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static final int NUM_RS = 2;
private static final String STRING_TABLE_NAME = "test";
private static final byte[] TEST_FAM = Bytes.toBytes("fam");
private static final TableName TABLE_NAME =
protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
protected static final TableName TABLE_NAME =
TableName.valueOf(STRING_TABLE_NAME);
/**
@ -93,7 +94,13 @@ public class TestSnapshotFromClient {
@Before
public void setup() throws Exception {
UTIL.createTable(TABLE_NAME, TEST_FAM);
HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
htd.setRegionReplication(getNumReplicas());
UTIL.createTable(htd, new byte[][]{TEST_FAM}, UTIL.getConfiguration());
}
protected int getNumReplicas() {
return 1;
}
@After

View File

@ -0,0 +1,30 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.client;
import org.apache.hadoop.hbase.LargeTests;
import org.junit.experimental.categories.Category;
@Category(LargeTests.class)
public class TestSnapshotFromClientWithRegionReplicas extends TestSnapshotFromClient {
@Override
protected int getNumReplicas() {
return 3;
}
}

View File

@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -48,6 +49,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterFileSystem;
@ -223,6 +225,8 @@ public class SnapshotTestingUtils {
// check the region snapshot for all the regions
List<HRegionInfo> regions = admin.getTableRegions(tableName);
// remove the non-default regions
RegionReplicaUtil.removeNonDefaultRegions(regions);
assertEquals(regions.size(), regionManifests.size());
// Verify Regions (redundant check, see MasterSnapshotVerifier)
@ -629,19 +633,32 @@ public class SnapshotTestingUtils {
}
public static void createTable(final HBaseTestingUtility util, final TableName tableName,
final byte[]... families) throws IOException, InterruptedException {
int regionReplication, final byte[]... families) throws IOException, InterruptedException {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setRegionReplication(regionReplication);
for (byte[] family: families) {
HColumnDescriptor hcd = new HColumnDescriptor(family);
htd.addFamily(hcd);
}
byte[][] splitKeys = getSplitKeys();
util.getHBaseAdmin().createTable(htd, splitKeys);
waitForTableToBeOnline(util, tableName);
assertEquals((splitKeys.length + 1) * regionReplication,
util.getHBaseAdmin().getTableRegions(tableName).size());
}
public static byte[][] getSplitKeys() {
byte[][] splitKeys = new byte[KEYS.length-2][];
byte[] hex = Bytes.toBytes("123456789abcde");
for (int i = 0; i < splitKeys.length; ++i) {
splitKeys[i] = new byte[] { KEYS[i+1] };
}
util.getHBaseAdmin().createTable(htd, splitKeys);
waitForTableToBeOnline(util, tableName);
assertEquals(KEYS.length-1, util.getHBaseAdmin().getTableRegions(tableName).size());
return splitKeys;
}
public static void createTable(final HBaseTestingUtility util, final TableName tableName,
final byte[]... families) throws IOException, InterruptedException {
createTable(util, tableName, 1, families);
}
public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
@ -711,4 +728,20 @@ public class SnapshotTestingUtils {
table.close();
}
}
public static void verifyReplicasCameOnline(TableName tableName, HBaseAdmin admin,
int regionReplication) throws IOException {
List<HRegionInfo> regions = admin.getTableRegions(tableName);
HashSet<HRegionInfo> set = new HashSet<HRegionInfo>();
for (HRegionInfo hri : regions) {
set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
for (int i = 0; i < regionReplication; i++) {
HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
if (!regions.contains(replica)) {
Assert.fail(replica + " is not contained in the list of online regions");
}
}
}
assert(set.size() == getSplitKeys().length + 1);
}
}