From a09c0c88b694af25575fb50812e14fb667f5ac5b Mon Sep 17 00:00:00 2001 From: ravowlga123 Date: Sat, 14 Sep 2019 19:16:49 +0200 Subject: [PATCH] HBASE-22899 logging improvements for snapshot operations w/large manifests (#547) Signed-off-by: Jan Hentschel --- .../hbase/snapshot/RestoreSnapshotHelper.java | 21 ++++++++++++------- .../hbase/snapshot/SnapshotManifest.java | 9 +++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index f533af4a7e5..43d892f3476 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -501,6 +501,7 @@ public class RestoreSnapshotHelper { getRegionHFileReferences(regionManifest); String tableName = tableDesc.getTableName().getNameAsString(); + final String snapshotName = snapshotDesc.getName(); // Restore families present in the table for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) { @@ -523,20 +524,21 @@ public class RestoreSnapshotHelper { // Remove hfiles not present in the snapshot for (String hfileName: familyFiles) { Path hfile = new Path(familyDir, hfileName); - LOG.trace("Removing hfile=" + hfileName + + LOG.trace("Removing HFile=" + hfileName + " not present in snapshot=" + snapshotName+ " from region=" + regionInfo.getEncodedName() + " table=" + tableName); HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile); } // Restore Missing files for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) { - LOG.debug("Adding HFileLink " + storeFile.getName() + - " to region=" + regionInfo.getEncodedName() + " table=" + tableName); + LOG.debug("Restoring missing HFileLink " + storeFile.getName() + + " of snapshot=" + snapshotName+ + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); } } else { // Family doesn't exists in the snapshot - LOG.trace("Removing family=" + Bytes.toString(family) + + LOG.trace("Removing family=" + Bytes.toString(family) + " in snapshot=" + snapshotName + " from region=" + regionInfo.getEncodedName() + " table=" + tableName); HFileArchiver.archiveFamilyByFamilyDir(fs, conf, regionInfo, familyDir, family); fs.delete(familyDir, true); @@ -552,7 +554,8 @@ public class RestoreSnapshotHelper { } for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) { - LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); + LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + + " of snapshot " + snapshotName + " to table=" + tableName); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); } } @@ -584,6 +587,7 @@ public class RestoreSnapshotHelper { if (regions == null || regions.isEmpty()) return null; final Map snapshotRegions = new HashMap<>(regions.size()); + final String snapshotName = snapshotDesc.getName(); // clone region info (change embedded tableName with the new one) RegionInfo[] clonedRegionsInfo = new RegionInfo[regions.size()]; @@ -596,7 +600,8 @@ public class RestoreSnapshotHelper { String snapshotRegionName = snapshotRegionInfo.getEncodedName(); String clonedRegionName = clonedRegionsInfo[i].getEncodedName(); regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName)); - LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName); + LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName + + " in snapshot " + snapshotName); // Add mapping between cloned region name and snapshot region info snapshotRegions.put(clonedRegionName, snapshotRegionInfo); @@ -640,10 +645,12 @@ public class RestoreSnapshotHelper { private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { final String tableName = tableDesc.getTableName().getNameAsString(); + final String snapshotName = snapshotDesc.getName(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { - LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); + LOG.info("Adding HFileLink " + storeFile.getName() +" from cloned region " + + "in snapshot " + snapshotName + " to table=" + tableName); restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index e2236ee06d2..bf6699d5309 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -185,7 +185,8 @@ public final class SnapshotManifest { @VisibleForTesting protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor) throws IOException { // 1. dump region meta info into the snapshot directory - LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot."); + final String snapshotName = desc.getName(); + LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot=" + snapshotName); Object regionData = visitor.regionOpen(regionInfo); monitor.rethrowException(); @@ -232,7 +233,8 @@ public final class SnapshotManifest { @VisibleForTesting protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOException { // 1. dump region meta info into the snapshot directory - LOG.debug("Storing '" + region + "' region-info for snapshot."); + final String snapshotName = desc.getName(); + LOG.debug("Storing '" + region + "' region-info for snapshot=" + snapshotName); Object regionData = visitor.regionOpen(region.getRegionInfo()); monitor.rethrowException(); @@ -256,7 +258,8 @@ public final class SnapshotManifest { monitor.rethrowException(); // create "reference" to this store file. - LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath()); + LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath() + + " for snapshot=" + snapshotName); visitor.storeFile(regionData, familyData, storeFile.getFileInfo()); } visitor.familyClose(regionData, familyData);