diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java index c23974c047c..c863b26ad53 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java @@ -146,6 +146,38 @@ public class HFileArchiver { + "), cannot delete region directory."); } + /** + * Remove from the specified region the store files of the specified column family, + * either by archiving them or outright deletion + * @param fs the filesystem where the store files live + * @param conf {@link Configuration} to examine to determine the archive directory + * @param parent Parent region hosting the store files + * @param tableDir {@link Path} to where the table is being stored (for building the archive path) + * @param family the family hosting the store files + * @throws IOException if the files could not be correctly disposed. + */ + public static void archiveFamily(FileSystem fs, Configuration conf, + HRegionInfo parent, Path tableDir, byte[] family) throws IOException { + Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); + FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir); + if (storeFiles == null) { + LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() + + ", family=" + Bytes.toString(family)); + return; + } + + FileStatusConverter getAsFile = new FileStatusConverter(fs); + Collection toArchive = Lists.transform(Arrays.asList(storeFiles), getAsFile); + Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, tableDir, family); + + // do the actual archive + if (!resolveAndArchive(fs, storeArchiveDir, toArchive)) { + throw new IOException("Failed to archive/delete all the files for region:" + + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) + + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); + } + } + /** * Remove the store files, either by archiving them or outright deletion * @param fs the filesystem where the store files live @@ -196,7 +228,7 @@ public class HFileArchiver { if (!resolveAndArchive(fs, storeArchiveDir, storeFiles)) { throw new IOException("Failed to archive/delete all the files for region:" + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family) - + " into " + storeArchiveDir + "Something is probably arwy on the filesystem."); + + " into " + storeArchiveDir + ". Something is probably awry on the filesystem."); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 48714137f95..398b66e6284 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -458,10 +458,14 @@ public class MasterFileSystem { public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName) throws IOException { - Path delDir = new Path(rootdir, - new Path(region.getTableNameAsString(), new Path( - region.getEncodedName(), new Path(Bytes.toString(familyName))))); - if (fs.delete(delDir, true) == false) { + // archive family store files + Path tableDir = new Path(rootdir, region.getTableNameAsString()); + HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName); + + // delete the family folder + Path familyDir = new Path(tableDir, + new Path(region.getEncodedName(), Bytes.toString(familyName))); + if (fs.delete(familyDir, true) == false) { throw new IOException("Could not delete family " + Bytes.toString(familyName) + " from FileSystem for region " + region.getRegionNameAsString() + "(" + region.getEncodedName() diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java index bc2a793bd6d..e1dd0c5ddb4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java @@ -230,15 +230,7 @@ public class TestHFileArchiving { // then get the current store files Path regionDir = region.getRegionDir(); - List storeFiles = getAllFileNames(fs, regionDir); - // remove all the non-storefile named files for the region - for (int i = 0; i < storeFiles.size(); i++) { - String file = storeFiles.get(i); - if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) { - storeFiles.remove(i--); - } - } - storeFiles.remove(HRegion.REGIONINFO_FILE); + List storeFiles = getRegionStoreFiles(fs, regionDir); // then delete the table so the hfiles get archived UTIL.deleteTable(TABLE_NAME); @@ -262,6 +254,66 @@ public class TestHFileArchiving { archivedFiles.containsAll(storeFiles)); } + /** + * Test that the store files are archived when a column family is removed. + * @throws Exception + */ + @Test + public void testArchiveOnTableFamilyDelete() throws Exception { + List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); + // make sure we only have 1 region serving this table + assertEquals(1, servingRegions.size()); + HRegion region = servingRegions.get(0); + + // get the parent RS and monitor + HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME); + FileSystem fs = hrs.getFileSystem(); + + // put some data on the region + LOG.debug("-------Loading table"); + UTIL.loadRegion(region, TEST_FAM); + + // get the hfiles in the region + List regions = hrs.getOnlineRegions(TABLE_NAME); + assertEquals("More that 1 region for test table.", 1, regions.size()); + + region = regions.get(0); + // wait for all the compactions to complete + region.waitForFlushesAndCompactions(); + + // disable table to prevent new updates + UTIL.getHBaseAdmin().disableTable(TABLE_NAME); + LOG.debug("Disabled table"); + + // remove all the files from the archive to get a fair comparison + clearArchiveDirectory(); + + // then get the current store files + Path regionDir = region.getRegionDir(); + List storeFiles = getRegionStoreFiles(fs, regionDir); + + // then delete the table so the hfiles get archived + UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM); + + // then get the files in the archive directory. + Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()); + List archivedFiles = getAllFileNames(fs, archiveDir); + Collections.sort(storeFiles); + Collections.sort(archivedFiles); + + LOG.debug("Store files:"); + for (int i = 0; i < storeFiles.size(); i++) { + LOG.debug(i + " - " + storeFiles.get(i)); + } + LOG.debug("Archive files:"); + for (int i = 0; i < archivedFiles.size(); i++) { + LOG.debug(i + " - " + archivedFiles.get(i)); + } + + assertTrue("Archived files are missing some of the store files!", + archivedFiles.containsAll(storeFiles)); + } + private void clearArchiveDirectory() throws IOException { UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true); } @@ -290,4 +342,18 @@ public class TestHFileArchiving { } return fileNames; } + + private List getRegionStoreFiles(final FileSystem fs, final Path regionDir) + throws IOException { + List storeFiles = getAllFileNames(fs, regionDir); + // remove all the non-storefile named files for the region + for (int i = 0; i < storeFiles.size(); i++) { + String file = storeFiles.get(i); + if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) { + storeFiles.remove(i--); + } + } + storeFiles.remove(HRegion.REGIONINFO_FILE); + return storeFiles; + } }