Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-12646 SnapshotInfo tool does not find mob data in snapshots

Browse Source 
Adds MobPath to HFileLink Resolver
This commit is contained in:
Jonathan M Hsieh 2014-12-06 08:47:14 -08:00
parent 33fc1918de
commit 30bc7d5598
2 changed files with 60 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
View File

@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.FSUtils;
@ -89,6 +90,7 @@ public class HFileLink extends FileLink {
private final Path archivePath;
private final Path originPath;
private final Path mobPath;
private final Path tempPath;
/**
@ -103,14 +105,26 @@ public class HFileLink extends FileLink {
/**
* @param rootDir Path to the root directory where hbase files are stored
* @param archiveDir Path to the hbase archive directory
* @param mobDir path to the hbase mob directory
* @param path The path of the HFile Link.
*/
public HFileLink(final Path rootDir, final Path archiveDir, final Path mobDir, final Path path) {
Path hfilePath = getRelativeTablePath(path);
this.tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath);
this.originPath = new Path(rootDir, hfilePath);
this.mobPath = new Path(mobDir, hfilePath);
this.archivePath = new Path(archiveDir, hfilePath);
setLocations(originPath, mobPath, tempPath, archivePath);
}
/**
* @param rootDir Path to the root directory where hbase files are stored
* @param archiveDir Path to the hbase archive directory
* @param path The path of the HFile Link.
*/
public HFileLink(final Path rootDir, final Path archiveDir, final Path path) {
Path hfilePath = getRelativeTablePath(path);
this.tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath);
this.originPath = new Path(rootDir, hfilePath);
this.archivePath = new Path(archiveDir, hfilePath);
setLocations(originPath, tempPath, archivePath);
this(rootDir, archiveDir, new Path(rootDir, MobConstants.MOB_DIR_NAME), path);
}
/**
@ -159,6 +173,11 @@ public class HFileLink extends FileLink {
}
/**
* @return the path of the mob hfiles.
*/
public Path getMobPath() { return this.mobPath; }
/**
* @param path Path to check.
* @return True if the path is a HFileLink.
*/

52
hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
View File

@ -115,14 +115,16 @@ public final class SnapshotInfo extends Configured implements Tool {
}
}
private AtomicInteger hfileArchiveCount = new AtomicInteger();
private AtomicInteger hfilesArchiveCount = new AtomicInteger();
private AtomicInteger hfilesCorrupted = new AtomicInteger();
private AtomicInteger hfilesMissing = new AtomicInteger();
private AtomicInteger hfilesCount = new AtomicInteger();
private AtomicInteger hfilesMobCount = new AtomicInteger();
private AtomicInteger logsMissing = new AtomicInteger();
private AtomicInteger logsCount = new AtomicInteger();
private AtomicLong hfileArchiveSize = new AtomicLong();
private AtomicLong hfileSize = new AtomicLong();
private AtomicLong hfilesArchiveSize = new AtomicLong();
private AtomicLong hfilesSize = new AtomicLong();
private AtomicLong hfilesMobSize = new AtomicLong();
private AtomicLong logSize = new AtomicLong();
private final SnapshotDescription snapshot;
@ -152,14 +154,17 @@ public final class SnapshotInfo extends Configured implements Tool {
/** @return the number of available store files */
public int getStoreFilesCount() {
return hfilesCount.get() + hfileArchiveCount.get();
return hfilesCount.get() + hfilesArchiveCount.get() + hfilesMobCount.get();
}
/** @return the number of available store files in the archive */
public int getArchivedStoreFilesCount() {
return hfileArchiveCount.get();
return hfilesArchiveCount.get();
}
/** @return the number of available store files in the mob dir */
public int getMobStoreFilesCount() { return hfilesMobCount.get(); }
/** @return the number of available log files */
public int getLogsCount() {
return logsCount.get();
@ -182,22 +187,30 @@ public final class SnapshotInfo extends Configured implements Tool {
/** @return the total size of the store files referenced by the snapshot */
public long getStoreFilesSize() {
return hfileSize.get() + hfileArchiveSize.get();
return hfilesSize.get() + hfilesArchiveSize.get() + hfilesMobSize.get();
}
/** @return the total size of the store files shared */
public long getSharedStoreFilesSize() {
return hfileSize.get();
return hfilesSize.get();
}
/** @return the total size of the store files in the archive */
public long getArchivedStoreFileSize() {
return hfileArchiveSize.get();
return hfilesArchiveSize.get();
}
/** @return the total size of the store files in the mob store*/
public long getMobStoreFilesSize() { return hfilesMobSize.get(); }
/** @return the percentage of the shared store files */
public float getSharedStoreFilePercentage() {
return ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100;
return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100;
}
/** @return the percentage of the mob store files */
public float getMobStoreFilePercentage() {
return ((float) hfilesMobSize.get() / (getStoreFilesSize())) * 100;
}
/** @return the total log size */
@ -209,7 +222,7 @@ public final class SnapshotInfo extends Configured implements Tool {
* Add the specified store file to the stats
* @param region region encoded Name
* @param family family name
* @param hfile store file name
* @param storeFile store file name
* @return the store file information
*/
FileInfo addStoreFile(final HRegionInfo region, final String family,
@ -222,11 +235,15 @@ public final class SnapshotInfo extends Configured implements Tool {
try {
if ((inArchive = fs.exists(link.getArchivePath()))) {
size = fs.getFileStatus(link.getArchivePath()).getLen();
hfileArchiveSize.addAndGet(size);
hfileArchiveCount.incrementAndGet();
hfilesArchiveSize.addAndGet(size);
hfilesArchiveCount.incrementAndGet();
} else if (inArchive = fs.exists(link.getMobPath())) {
size = fs.getFileStatus(link.getMobPath()).getLen();
hfilesMobSize.addAndGet(size);
hfilesMobCount.incrementAndGet();
} else {
size = link.getFileStatus(fs).getLen();
hfileSize.addAndGet(size);
hfilesSize.addAndGet(size);
hfilesCount.incrementAndGet();
}
isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size);
@ -442,11 +459,14 @@ public final class SnapshotInfo extends Configured implements Tool {
}
if (showStats) {
System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n",
stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(),
System.out.printf("%d HFiles (%d in archive, %d in mob storage), total size %s " +
"(%.2f%% %s shared with the source table, %.2f%% %s in mob dir)%n",
stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), stats.getMobStoreFilesCount(),
fileSizeToString(stats.getStoreFilesSize()),
stats.getSharedStoreFilePercentage(),
fileSizeToString(stats.getSharedStoreFilesSize())
fileSizeToString(stats.getSharedStoreFilesSize()),
stats.getMobStoreFilePercentage(),
fileSizeToString(stats.getMobStoreFilesSize())
);
System.out.printf("%d Logs, total size %s%n",
stats.getLogsCount(), fileSizeToString(stats.getLogsSize()));