Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14302 TableSnapshotInputFormat should not create back references when restoring snapshot 

Conflicts:
	hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
	hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
	hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
This commit is contained in:
Enis Soztutar 2015-08-25 12:10:15 -07:00
parent 64f9d43d01
commit 65a96b4883
5 changed files with 174 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
View File

@ -308,9 +308,30 @@ public class HFileLink extends FileLink {
public static boolean create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
final String hfileName) throws IOException {
return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true);
}
/**
* Create a new HFileLink
*
* <p>It also adds a back-reference to the hfile back-reference directory
* to simplify the reference-count and the cleaning process.
*
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param hfileRegionInfo - Linked HFile Region Info
* @param hfileName - Linked HFile name
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return true if the file is created, otherwise the file exists.
* @throws IOException on file or parent directory creation failure
*/
public static boolean create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final HRegionInfo hfileRegionInfo,
final String hfileName, final boolean createBackRef) throws IOException {
TableName linkedTable = hfileRegionInfo.getTable();
String linkedRegion = hfileRegionInfo.getEncodedName();
return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName);
return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef);
}
/**
@ -331,6 +352,28 @@ public class HFileLink extends FileLink {
public static boolean create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
final String hfileName) throws IOException {
return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true);
}
/**
* Create a new HFileLink
*
* <p>It also adds a back-reference to the hfile back-reference directory
* to simplify the reference-count and the cleaning process.
*
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param linkedTable - Linked Table Name
* @param linkedRegion - Linked Region Name
* @param hfileName - Linked HFile name
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return true if the file is created, otherwise the file exists.
* @throws IOException on file or parent directory creation failure
*/
public static boolean create(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
final String hfileName, final boolean createBackRef) throws IOException {
String familyName = dstFamilyPath.getName();
String regionName = dstFamilyPath.getParent().getName();
String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent())
@ -345,19 +388,24 @@ public class HFileLink extends FileLink {
// Make sure the FileLink reference directory exists
Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
linkedTable, linkedRegion, familyName);
Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
fs.mkdirs(backRefssDir);
Path backRefPath = null;
if (createBackRef) {
Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
fs.mkdirs(backRefssDir);
// Create the reference for the link
Path backRefPath = new Path(backRefssDir, refName);
fs.createNewFile(backRefPath);
// Create the reference for the link
backRefPath = new Path(backRefssDir, refName);
fs.createNewFile(backRefPath);
}
try {
// Create the link
return fs.createNewFile(new Path(dstFamilyPath, name));
} catch (IOException e) {
LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
// Revert the reference if the link creation failed
fs.delete(backRefPath, false);
if (createBackRef) {
fs.delete(backRefPath, false);
}
throw e;
}
}
@ -376,13 +424,34 @@ public class HFileLink extends FileLink {
* @throws IOException on file or parent directory creation failure
*/
public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final String hfileLinkName) throws IOException {
final Path dstFamilyPath, final String hfileLinkName)
throws IOException {
return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true);
}
/**
* Create a new HFileLink starting from a hfileLink name
*
* <p>It also adds a back-reference to the hfile back-reference directory
* to simplify the reference-count and the cleaning process.
*
* @param conf {@link Configuration} to read for the archive directory name
* @param fs {@link FileSystem} on which to write the HFileLink
* @param dstFamilyPath - Destination path (table/region/cf/)
* @param hfileLinkName - HFileLink name (it contains hfile-region-table)
* @param createBackRef - Whether back reference should be created. Defaults to true.
* @return true if the file is created, otherwise the file exists.
* @throws IOException on file or parent directory creation failure
*/
public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef)
throws IOException {
Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
if (!m.matches()) {
throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
}
return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)),
m.group(3), m.group(4));
m.group(3), m.group(4), createBackRef);
}
/**

35
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
View File

@ -127,6 +127,7 @@ public class RestoreSnapshotHelper {
private final Configuration conf;
private final FileSystem fs;
private final boolean createBackRefs;
public RestoreSnapshotHelper(final Configuration conf,
final FileSystem fs,
@ -134,7 +135,18 @@ public class RestoreSnapshotHelper {
final HTableDescriptor tableDescriptor,
final Path rootDir,
final ForeignExceptionDispatcher monitor,
final MonitoredTask status)
final MonitoredTask status) {
this(conf, fs, manifest, tableDescriptor, rootDir, monitor, status, true);
}
public RestoreSnapshotHelper(final Configuration conf,
final FileSystem fs,
final SnapshotManifest manifest,
final HTableDescriptor tableDescriptor,
final Path rootDir,
final ForeignExceptionDispatcher monitor,
final MonitoredTask status,
final boolean createBackRefs)
{
this.fs = fs;
this.conf = conf;
@ -146,6 +158,7 @@ public class RestoreSnapshotHelper {
this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
this.monitor = monitor;
this.status = status;
this.createBackRefs = createBackRefs;
}
/**
@ -434,7 +447,7 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
LOG.debug("Adding HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
} else {
// Family doesn't exists in the snapshot
@ -455,7 +468,7 @@ public class RestoreSnapshotHelper {
for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
}
}
@ -538,7 +551,7 @@ public class RestoreSnapshotHelper {
Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
restoreStoreFile(familyDir, snapshotRegionInfo, storeFile);
restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
}
}
}
@ -553,17 +566,19 @@ public class RestoreSnapshotHelper {
* </ul>
* @param familyDir destination directory for the store file
* @param regionInfo destination region info for the table
* @param hfileName store file name (can be a Reference, HFileLink or simple HFile)
* @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
* @param createBackRef - Whether back reference should be created. Defaults to true.
*/
private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
throws IOException {
String hfileName = storeFile.getName();
if (HFileLink.isHFileLink(hfileName)) {
HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
} else if (StoreFileInfo.isReference(hfileName)) {
restoreReferenceFile(familyDir, regionInfo, storeFile);
} else {
HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
}
}
@ -729,8 +744,10 @@ public class RestoreSnapshotHelper {
"Restoring snapshot '" + snapshotName + "' to directory " + restoreDir);
ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
// we send createBackRefs=false so that restored hfiles do not create back reference links
// in the base hbase root dir.
RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs,
manifest, manifest.getTableDescriptor(), restoreDir, monitor, status);
manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false);
helper.restoreHdfsRegions(); // TODO: parallelize.
if (LOG.isDebugEnabled()) {

9
hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
View File

@ -147,6 +147,15 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
testWithMapReduce(UTIL, "testWithMapReduceAndOfflineHBaseMultiRegion", 10, 10, true);
}
@Override
public void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
String snapshotName, Path tmpTableDir) throws Exception {
JobConf job = new JobConf(UTIL.getConfiguration());
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName,
COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
NullWritable.class, job, false, tmpTableDir);
}
@Override
protected void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits) throws Exception {

51
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.mapreduce;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
@ -28,14 +29,20 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.junit.Assert;
import org.junit.Test;
import static org.junit.Assert.assertFalse;
import java.io.IOException;
import java.util.Arrays;
@ -101,6 +108,50 @@ public abstract class TableSnapshotInputFormatTestBase {
testWithMapReduce(UTIL, "testWithMapReduceAndOfflineHBaseMultiRegion", 10, 8, true);
}
// Test that snapshot restore does not create back references in the HBase root dir.
@Test
public void testRestoreSnapshotDoesNotCreateBackRefLinks() throws Exception {
setupCluster();
TableName tableName = TableName.valueOf("testRestoreSnapshotDoesNotCreateBackRefLinks");
String snapshotName = "foo";
try {
createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);
Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName);
testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);
Path rootDir = FSUtils.getRootDir(UTIL.getConfiguration());
for (Path regionDir : FSUtils.getRegionDirs(fs, FSUtils.getTableDir(rootDir, tableName))) {
for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) {
for (FileStatus status : fs.listStatus(storeDir)) {
System.out.println(status.getPath());
if (StoreFileInfo.isValid(status)) {
Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
tableName, regionDir.getName(), storeDir.getName());
Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName());
// assert back references directory is empty
assertFalse("There is a back reference in " + path, fs.exists(path));
path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName());
// assert back references directory is empty
assertFalse("There is a back reference in " + path, fs.exists(path));
}
}
}
}
} finally {
UTIL.getHBaseAdmin().deleteSnapshot(snapshotName);
UTIL.deleteTable(tableName);
tearDownCluster();
}
}
public abstract void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
String snapshotName, Path tmpTableDir) throws Exception;
protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
setupCluster();

10
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
View File

@ -172,6 +172,16 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
}
}
@Override
public void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName tableName,
String snapshotName, Path tmpTableDir) throws Exception {
Job job = new Job(UTIL.getConfiguration());
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
new Scan(), TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
NullWritable.class, job, false, tmpTableDir);
}
@Override
public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits) throws Exception {
setupCluster();