Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22899 logging improvements for snapshot operations w/large manifests (#547) 

Signed-off-by: Jan Hentschel <jan.hentschel@ultratendency.com>
This commit is contained in:
ravowlga123 2019-09-14 19:16:49 +02:00 committed by Michael Stack
parent 3583ef7476
commit a09c0c88b6
2 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
View File

@ -501,6 +501,7 @@ public class RestoreSnapshotHelper {
getRegionHFileReferences(regionManifest);
String tableName = tableDesc.getTableName().getNameAsString();
final String snapshotName = snapshotDesc.getName();
// Restore families present in the table
for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
@ -523,20 +524,21 @@ public class RestoreSnapshotHelper {
// Remove hfiles not present in the snapshot
for (String hfileName: familyFiles) {
Path hfile = new Path(familyDir, hfileName);
LOG.trace("Removing hfile=" + hfileName +
LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName+
" from region=" + regionInfo.getEncodedName() + " table=" + tableName);
HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
}
// Restore Missing files
for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
LOG.debug("Adding HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
LOG.debug("Restoring missing HFileLink " + storeFile.getName() +
" of snapshot=" + snapshotName+
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
} else {
// Family doesn't exists in the snapshot
LOG.trace("Removing family=" + Bytes.toString(family) +
LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName +
" from region=" + regionInfo.getEncodedName() + " table=" + tableName);
HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
fs.delete(familyDir, true);
@ -552,7 +554,8 @@ public class RestoreSnapshotHelper {
}
for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName()
+ " of snapshot " + snapshotName + " to table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
}
}
@ -584,6 +587,7 @@ public class RestoreSnapshotHelper {
if (regions == null || regions.isEmpty()) return null;
final Map<String, RegionInfo> snapshotRegions = new HashMap<>(regions.size());
final String snapshotName = snapshotDesc.getName();
// clone region info (change embedded tableName with the new one)
RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()];
@ -596,7 +600,8 @@ public class RestoreSnapshotHelper {
String snapshotRegionName = snapshotRegionInfo.getEncodedName();
String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName +
" in snapshot " + snapshotName);
// Add mapping between cloned region name and snapshot region info
snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
@ -640,10 +645,12 @@ public class RestoreSnapshotHelper {
private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo,
final SnapshotRegionManifest manifest) throws IOException {
final String tableName = tableDesc.getTableName().getNameAsString();
final String snapshotName = snapshotDesc.getName();
for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
LOG.info("Adding HFileLink " + storeFile.getName() +" from cloned region "
+ "in snapshot " + snapshotName + " to table=" + tableName);
restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
}
}

9
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
View File

@ -185,7 +185,8 @@ public final class SnapshotManifest {
@VisibleForTesting
protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor) throws IOException {
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot.");
final String snapshotName = desc.getName();
LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(regionInfo);
monitor.rethrowException();
@ -232,7 +233,8 @@ public final class SnapshotManifest {
@VisibleForTesting
protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOException {
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing '" + region + "' region-info for snapshot.");
final String snapshotName = desc.getName();
LOG.debug("Storing '" + region + "' region-info for snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(region.getRegionInfo());
monitor.rethrowException();
@ -256,7 +258,8 @@ public final class SnapshotManifest {
monitor.rethrowException();
// create "reference" to this store file.
LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath());
LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath() +
" for snapshot=" + snapshotName);
visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
}
visitor.familyClose(regionData, familyData);