Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22899 logging improvements for snapshot operations w/large manifests (#547) 

Signed-off-by: Jan Hentschel <jan.hentschel@ultratendency.com>
This commit is contained in:
ravowlga123 2019-09-14 19:16:49 +02:00 committed by Michael Stack
parent 3583ef7476
commit a09c0c88b6
2 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
View File

@ -501,6 +501,7 @@ public class RestoreSnapshotHelper {
getRegionHFileReferences(regionManifest); getRegionHFileReferences(regionManifest);
String tableName = tableDesc.getTableName().getNameAsString(); String tableName = tableDesc.getTableName().getNameAsString();
final String snapshotName = snapshotDesc.getName();
// Restore families present in the table // Restore families present in the table
for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
@ -523,20 +524,21 @@ public class RestoreSnapshotHelper {
// Remove hfiles not present in the snapshot // Remove hfiles not present in the snapshot
for (String hfileName: familyFiles) { for (String hfileName: familyFiles) {
Path hfile = new Path(familyDir, hfileName); Path hfile = new Path(familyDir, hfileName);
LOG.trace("Removing hfile=" + hfileName + LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName+
" from region=" + regionInfo.getEncodedName() + " table=" + tableName); " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile); HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
} }
// Restore Missing files // Restore Missing files
for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) { for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
LOG.debug("Adding HFileLink " + storeFile.getName() + LOG.debug("Restoring missing HFileLink " + storeFile.getName() +
" to region=" + regionInfo.getEncodedName() + " table=" + tableName); " of snapshot=" + snapshotName+
" to region=" + regionInfo.getEncodedName() + " table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
} }
} else { } else {
// Family doesn't exists in the snapshot // Family doesn't exists in the snapshot
LOG.trace("Removing family=" + Bytes.toString(family) + LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName +
" from region=" + regionInfo.getEncodedName() + " table=" + tableName); " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family); HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family);
fs.delete(familyDir, true); fs.delete(familyDir, true);
@ -552,7 +554,8 @@ public class RestoreSnapshotHelper {
} }
for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) { for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName()
+ " of snapshot " + snapshotName + " to table=" + tableName);
restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
} }
} }
@ -584,6 +587,7 @@ public class RestoreSnapshotHelper {
if (regions == null || regions.isEmpty()) return null; if (regions == null || regions.isEmpty()) return null;
final Map<String, RegionInfo> snapshotRegions = new HashMap<>(regions.size()); final Map<String, RegionInfo> snapshotRegions = new HashMap<>(regions.size());
final String snapshotName = snapshotDesc.getName();
// clone region info (change embedded tableName with the new one) // clone region info (change embedded tableName with the new one)
RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()]; RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()];
@ -596,7 +600,8 @@ public class RestoreSnapshotHelper {
String snapshotRegionName = snapshotRegionInfo.getEncodedName(); String snapshotRegionName = snapshotRegionInfo.getEncodedName();
String clonedRegionName = clonedRegionsInfo[i].getEncodedName(); String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName)); regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName); LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName +
" in snapshot " + snapshotName);
// Add mapping between cloned region name and snapshot region info // Add mapping between cloned region name and snapshot region info
snapshotRegions.put(clonedRegionName, snapshotRegionInfo); snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
@ -640,10 +645,12 @@ public class RestoreSnapshotHelper {
private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo, private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo,
final SnapshotRegionManifest manifest) throws IOException { final SnapshotRegionManifest manifest) throws IOException {
final String tableName = tableDesc.getTableName().getNameAsString(); final String tableName = tableDesc.getTableName().getNameAsString();
final String snapshotName = snapshotDesc.getName();
for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); LOG.info("Adding HFileLink " + storeFile.getName() +" from cloned region "
+ "in snapshot " + snapshotName + " to table=" + tableName);
restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
} }
} }

9
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
View File

@ -185,7 +185,8 @@ public final class SnapshotManifest {
@VisibleForTesting @VisibleForTesting
protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor) throws IOException { protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor) throws IOException {
// 1. dump region meta info into the snapshot directory // 1. dump region meta info into the snapshot directory
LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot."); final String snapshotName = desc.getName();
LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(regionInfo); Object regionData = visitor.regionOpen(regionInfo);
monitor.rethrowException(); monitor.rethrowException();
@ -232,7 +233,8 @@ public final class SnapshotManifest {
@VisibleForTesting @VisibleForTesting
protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOException { protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOException {
// 1. dump region meta info into the snapshot directory // 1. dump region meta info into the snapshot directory
LOG.debug("Storing '" + region + "' region-info for snapshot."); final String snapshotName = desc.getName();
LOG.debug("Storing '" + region + "' region-info for snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(region.getRegionInfo()); Object regionData = visitor.regionOpen(region.getRegionInfo());
monitor.rethrowException(); monitor.rethrowException();
@ -256,7 +258,8 @@ public final class SnapshotManifest {
monitor.rethrowException(); monitor.rethrowException();
// create "reference" to this store file. // create "reference" to this store file.
LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath()); LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath() +
" for snapshot=" + snapshotName);
visitor.storeFile(regionData, familyData, storeFile.getFileInfo()); visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
} }
visitor.familyClose(regionData, familyData); visitor.familyClose(regionData, familyData);