Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-21300 Fix the wrong reference file path when restoring snapshots for tables with MOB columns 

Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
meiyi 2018-11-27 14:32:40 +08:00 committed by Guanghao Zhang
parent 7946f96035
commit 001aabd40a
5 changed files with 147 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
View File

@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MetricsSnapshot;
import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
@ -449,10 +450,38 @@ public class CloneSnapshotProcedure
// 3. Move Table temp directory to the hbase root location
CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir);
// Move Table temp mob directory to the hbase root location
Path tempMobTableDir = MobUtils.getMobTableDir(tempdir, tableDescriptor.getTableName());
if (mfs.getFileSystem().exists(tempMobTableDir)) {
moveTempMobDirectoryToHBaseRoot(mfs, tableDescriptor, tempMobTableDir);
}
return newRegions;
}
/**
* Move table temp mob directory to the hbase root location
* @param mfs The master file system
* @param tableDescriptor The table to operate on
* @param tempMobTableDir The temp mob directory of table
* @throws IOException If failed to move temp mob dir to hbase root dir
*/
private void moveTempMobDirectoryToHBaseRoot(final MasterFileSystem mfs,
final TableDescriptor tableDescriptor, final Path tempMobTableDir) throws IOException {
FileSystem fs = mfs.getFileSystem();
final Path tableMobDir =
MobUtils.getMobTableDir(mfs.getRootDir(), tableDescriptor.getTableName());
if (!fs.delete(tableMobDir, true) && fs.exists(tableMobDir)) {
throw new IOException("Couldn't delete mob table " + tableMobDir);
}
if (!fs.exists(tableMobDir.getParent())) {
fs.mkdirs(tableMobDir.getParent());
}
if (!fs.rename(tempMobTableDir, tableMobDir)) {
throw new IOException("Unable to move mob table from temp=" + tempMobTableDir
+ " to hbase root=" + tableMobDir);
}
}
/**
* Add regions to hbase:meta table.
* @param env MasterProcedureEnv

38
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
View File

@ -367,7 +367,17 @@ public final class MobUtils {
*/
public static Path getMobHome(Configuration conf) {
Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
return new Path(hbaseDir, MobConstants.MOB_DIR_NAME);
return getMobHome(hbaseDir);
}
/**
* Gets the root dir of the mob files under the qualified HBase root dir.
* It's {rootDir}/mobdir.
* @param rootDir The qualified path of HBase root directory.
* @return The root dir of the mob file.
*/
public static Path getMobHome(Path rootDir) {
return new Path(rootDir, MobConstants.MOB_DIR_NAME);
}
/**
@ -383,15 +393,37 @@ public final class MobUtils {
return mobRootDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
}
/**
* Gets the table dir of the mob files under the qualified HBase root dir.
* It's {rootDir}/mobdir/data/${namespace}/${tableName}
* @param rootDir The qualified path of HBase root directory.
* @param tableName The name of table.
* @return The table dir of the mob file.
*/
public static Path getMobTableDir(Path rootDir, TableName tableName) {
return FSUtils.getTableDir(getMobHome(rootDir), tableName);
}
/**
* Gets the region dir of the mob files.
* It's {HBASE_DIR}/mobdir/{namespace}/{tableName}/{regionEncodedName}.
* It's {HBASE_DIR}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}.
* @param conf The current configuration.
* @param tableName The current table name.
* @return The region dir of the mob files.
*/
public static Path getMobRegionPath(Configuration conf, TableName tableName) {
Path tablePath = FSUtils.getTableDir(getMobHome(conf), tableName);
return getMobRegionPath(new Path(conf.get(HConstants.HBASE_DIR)), tableName);
}
/**
* Gets the region dir of the mob files under the specified root dir.
* It's {rootDir}/mobdir/data/{namespace}/{tableName}/{regionEncodedName}.
* @param rootDir The qualified path of HBase root directory.
* @param tableName The current table name.
* @return The region dir of the mob files.
*/
public static Path getMobRegionPath(Path rootDir, TableName tableName) {
Path tablePath = FSUtils.getTableDir(getMobHome(rootDir), tableName);
RegionInfo regionInfo = getMobRegionInfo(tableName);
return new Path(tablePath, regionInfo.getEncodedName());
}

2
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
View File

@ -620,7 +620,7 @@ public class RestoreSnapshotHelper {
private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
final RegionInfo region) throws IOException {
// clone region info (change embedded tableName with the new one)
Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
Path clonedRegionPath = MobUtils.getMobRegionPath(rootDir, tableDesc.getTableName());
cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
}

16
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java
View File

@ -20,9 +20,12 @@ package org.apache.hadoop.hbase.snapshot;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils.SnapshotMock;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.ClassRule;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
@ -31,7 +34,7 @@ import org.slf4j.LoggerFactory;
/**
* Test the restore/clone operation from a file-system point of view.
*/
@Category(SmallTests.class)
@Category(MediumTests.class)
public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper {
@ClassRule
@ -49,4 +52,13 @@ public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper {
protected SnapshotMock createSnapshotMock() throws IOException {
return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
}
@Override
protected void createTableAndSnapshot(TableName tableName, String snapshotName)
throws IOException {
byte[] column = Bytes.toBytes("A");
Table table = MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, column);
TEST_UTIL.loadTable(table, column);
TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
}
}

69
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java
View File

@ -24,22 +24,32 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@ -52,7 +62,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.Snapshot
/**
* Test the restore/clone operation from a file-system point of view.
*/
@Category({RegionServerTests.class, SmallTests.class})
@Category({RegionServerTests.class, MediumTests.class})
public class TestRestoreSnapshotHelper {
@ClassRule
@ -72,6 +82,16 @@ public class TestRestoreSnapshotHelper {
protected void setupConf(Configuration conf) {
}
@BeforeClass
public static void setupCluster() throws Exception {
TEST_UTIL.startMiniCluster();
}
@AfterClass
public static void tearDownCluster() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
@Before
public void setup() throws Exception {
rootDir = TEST_UTIL.getDataTestDir("testRestore");
@ -101,6 +121,51 @@ public class TestRestoreSnapshotHelper {
restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
}
@Test
public void testNoHFileLinkInRootDir() throws IOException {
rootDir = TEST_UTIL.getDefaultRootDirPath();
FSUtils.setRootDir(conf, rootDir);
fs = rootDir.getFileSystem(conf);
TableName tableName = TableName.valueOf("testNoHFileLinkInRootDir");
String snapshotName = tableName.getNameAsString() + "-snapshot";
createTableAndSnapshot(tableName, snapshotName);
Path restoreDir = new Path("/hbase/.tmp-restore");
RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
checkNoHFileLinkInTableDir(tableName);
}
protected void createTableAndSnapshot(TableName tableName, String snapshotName)
throws IOException {
byte[] column = Bytes.toBytes("A");
Table table = TEST_UTIL.createTable(tableName, column, 2);
TEST_UTIL.loadTable(table, column);
TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
}
private void checkNoHFileLinkInTableDir(TableName tableName) throws IOException {
Path[] tableDirs = new Path[] { CommonFSUtils.getTableDir(rootDir, tableName),
CommonFSUtils.getTableDir(new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY), tableName),
CommonFSUtils.getTableDir(MobUtils.getMobHome(rootDir), tableName) };
for (Path tableDir : tableDirs) {
Assert.assertFalse(hasHFileLink(tableDir));
}
}
private boolean hasHFileLink(Path tableDir) throws IOException {
if (fs.exists(tableDir)) {
RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(tableDir, true);
while (iterator.hasNext()) {
LocatedFileStatus fileStatus = iterator.next();
if (fileStatus.isFile() && HFileLink.isHFileLink(fileStatus.getPath())) {
return true;
}
}
}
return false;
}
private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException {
// Test Rolling-Upgrade like Snapshot.
// half machines writing using v1 and the others using v2 format.