diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java index ee0d0e2aee8..e332c94150c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -42,14 +42,11 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; +import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.Pair; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -62,7 +59,7 @@ import org.junit.rules.TestRule; /** * Test Export Snapshot Tool */ -@Category({VerySlowRegionServerTests.class, LargeTests.class}) +@Category({VerySlowMapReduceTests.class, LargeTests.class}) public class TestExportSnapshot { @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). withTimeout(this.getClass()).withLookingForStuckThread(true).build(); @@ -128,6 +125,14 @@ public class TestExportSnapshot { SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); } + protected interface RegionPredicate { + boolean evaluate(final HRegionInfo regionInfo); + } + + protected RegionPredicate getBypassRegionPredicate() { + return null; + } + @After public void tearDown() throws Exception { TEST_UTIL.deleteTable(tableName); @@ -135,61 +140,6 @@ public class TestExportSnapshot { SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); } - - /** - * Verfy the result of getBalanceSplits() method. - * The result are groups of files, used as input list for the "export" mappers. - * All the groups should have similar amount of data. - * - * The input list is a pair of file path and length. - * The getBalanceSplits() function sort it by length, - * and assign to each group a file, going back and forth through the groups. - */ - @Test - public void testBalanceSplit() throws Exception { - // Create a list of files - List> files = new ArrayList>(); - for (long i = 0; i <= 20; i++) { - SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() - .setType(SnapshotFileInfo.Type.HFILE) - .setHfile("file-" + i) - .build(); - files.add(new Pair(fileInfo, i)); - } - - // Create 5 groups (total size 210) - // group 0: 20, 11, 10, 1 (total size: 42) - // group 1: 19, 12, 9, 2 (total size: 42) - // group 2: 18, 13, 8, 3 (total size: 42) - // group 3: 17, 12, 7, 4 (total size: 42) - // group 4: 16, 11, 6, 5 (total size: 42) - List>> splits = ExportSnapshot.getBalancedSplits(files, 5); - assertEquals(5, splits.size()); - - String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"}; - verifyBalanceSplit(splits.get(0), split0, 42); - String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"}; - verifyBalanceSplit(splits.get(1), split1, 42); - String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"}; - verifyBalanceSplit(splits.get(2), split2, 42); - String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"}; - verifyBalanceSplit(splits.get(3), split3, 42); - String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"}; - verifyBalanceSplit(splits.get(4), split4, 42); - } - - private void verifyBalanceSplit(final List> split, - final String[] expected, final long expectedSize) { - assertEquals(expected.length, split.size()); - long totalSize = 0; - for (int i = 0; i < expected.length; ++i) { - Pair fileInfo = split.get(i); - assertEquals(expected[i], fileInfo.getFirst().getHfile()); - totalSize += fileInfo.getSecond(); - } - assertEquals(expectedSize, totalSize); - } - /** * Verify if exported snapshot and copied files matches the original one. */ @@ -201,7 +151,11 @@ public class TestExportSnapshot { @Test public void testExportFileSystemStateWithSkipTmp() throws Exception { TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); + try { + testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); + } finally { + TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); + } } @Test @@ -223,57 +177,28 @@ public class TestExportSnapshot { testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); } - /** - * Mock a snapshot with files in the archive dir, - * two regions, and one reference file. - */ - @Test - public void testSnapshotWithRefsExportFileSystemState() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - - Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - - SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1", - "tableWithRefsV1"); - testSnapshotWithRefsExportFileSystemState(builder); - - snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - builder = snapshotMock.createSnapshotV2("tableWithRefsV2", "tableWithRefsV2"); - testSnapshotWithRefsExportFileSystemState(builder); - } - - /** - * Generates a couple of regions for the specified SnapshotMock, - * and then it will run the export and verification. - */ - private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) - throws Exception { - Path[] r1Files = builder.addRegion(); - Path[] r2Files = builder.addRegion(); - builder.commit(); - int snapshotFilesCount = r1Files.length + r2Files.length; - - byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); - TableName tableName = builder.getTableDescriptor().getTableName(); - testExportFileSystemState(tableName, snapshotName, snapshotName, snapshotFilesCount); - } - private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, final byte[] targetName, int filesExpected) throws Exception { - Path copyDir = getHdfsDestinationDir(); - testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, false); - removeExportDir(copyDir); + testExportFileSystemState(tableName, snapshotName, targetName, + filesExpected, getHdfsDestinationDir(), false); + } + + protected void testExportFileSystemState(final TableName tableName, + final byte[] snapshotName, final byte[] targetName, int filesExpected, + Path copyDir, boolean overwrite) throws Exception { + testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, + filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, + overwrite, getBypassRegionPredicate()); } /** * Test ExportSnapshot */ - private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite) - throws Exception { - URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri(); + protected static void testExportFileSystemState(final Configuration conf, final TableName tableName, + final byte[] snapshotName, final byte[] targetName, final int filesExpected, + final Path sourceDir, Path copyDir, final boolean overwrite, + final RegionPredicate bypassregionPredicate) throws Exception { + URI hdfsUri = FileSystem.get(conf).getUri(); FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); copyDir = copyDir.makeQualified(fs); @@ -289,8 +214,7 @@ public class TestExportSnapshot { if (overwrite) opts.add("-overwrite"); // Export Snapshot - int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), - opts.toArray(new String[opts.size()])); + int res = ExportSnapshot.innerMain(conf, opts.toArray(new String[opts.size()])); assertEquals(0, res); // Verify File-System state @@ -304,12 +228,13 @@ public class TestExportSnapshot { } // compare the snapshot metadata and verify the hfiles - final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration()); + final FileSystem hdfs = FileSystem.get(hdfsUri, conf); final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); - verifySnapshotDir(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir), + verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), fs, new Path(copyDir, targetDir)); - Set snapshotFiles = verifySnapshot(fs, copyDir, tableName, Bytes.toString(targetName)); + Set snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName, + Bytes.toString(targetName), bypassregionPredicate); assertEquals(filesExpected, snapshotFiles.size()); } @@ -358,30 +283,33 @@ public class TestExportSnapshot { /* * verify if the snapshot folder on file-system 1 match the one on file-system 2 */ - private void verifySnapshotDir(final FileSystem fs1, final Path root1, + protected static void verifySnapshotDir(final FileSystem fs1, final Path root1, final FileSystem fs2, final Path root2) throws IOException { assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); } - protected boolean bypassRegion(HRegionInfo regionInfo) { - return false; + protected Set verifySnapshot(final FileSystem fs, final Path rootDir, + final TableName tableName, final String snapshotName) throws IOException { + return verifySnapshot(TEST_UTIL.getConfiguration(), fs, rootDir, tableName, + snapshotName, getBypassRegionPredicate()); } /* * Verify if the files exists */ - private Set verifySnapshot(final FileSystem fs, final Path rootDir, - final TableName tableName, final String snapshotName) throws IOException { + protected static Set verifySnapshot(final Configuration conf, final FileSystem fs, + final Path rootDir, final TableName tableName, final String snapshotName, + final RegionPredicate bypassregionPredicate) throws IOException { final Path exportedSnapshot = new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); final Set snapshotFiles = new HashSet(); final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); - SnapshotReferenceUtil.visitReferencedFiles(TEST_UTIL.getConfiguration(), fs, exportedSnapshot, + SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, new SnapshotReferenceUtil.SnapshotVisitor() { @Override public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { - if (bypassRegion(regionInfo)) + if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) return; String hfile = storeFile.getName(); @@ -408,10 +336,10 @@ public class TestExportSnapshot { return snapshotFiles; } - private Set listFiles(final FileSystem fs, final Path root, final Path dir) + private static Set listFiles(final FileSystem fs, final Path root, final Path dir) throws IOException { Set files = new HashSet(); - int rootPrefix = root.toString().length(); + int rootPrefix = root.makeQualified(fs).toString().length(); FileStatus[] list = FSUtils.listStatus(fs, dir); if (list != null) { for (FileStatus fstat: list) { @@ -419,7 +347,7 @@ public class TestExportSnapshot { if (fstat.isDirectory()) { files.addAll(listFiles(fs, root, fstat.getPath())); } else { - files.add(fstat.getPath().toString().substring(rootPrefix)); + files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix)); } } } @@ -439,7 +367,7 @@ public class TestExportSnapshot { return path; } - private void removeExportDir(final Path path) throws IOException { + private static void removeExportDir(final Path path) throws IOException { FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); fs.delete(path, true); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java new file mode 100644 index 00000000000..2d0088bcccd --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotHelpers.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test Export Snapshot Tool helpers + */ +@Category({RegionServerTests.class, SmallTests.class}) +public class TestExportSnapshotHelpers { + private static final Log LOG = LogFactory.getLog(TestExportSnapshotHelpers.class); + + /** + * Verfy the result of getBalanceSplits() method. + * The result are groups of files, used as input list for the "export" mappers. + * All the groups should have similar amount of data. + * + * The input list is a pair of file path and length. + * The getBalanceSplits() function sort it by length, + * and assign to each group a file, going back and forth through the groups. + */ + @Test + public void testBalanceSplit() throws Exception { + // Create a list of files + List> files = new ArrayList>(); + for (long i = 0; i <= 20; i++) { + SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() + .setType(SnapshotFileInfo.Type.HFILE) + .setHfile("file-" + i) + .build(); + files.add(new Pair(fileInfo, i)); + } + + // Create 5 groups (total size 210) + // group 0: 20, 11, 10, 1 (total size: 42) + // group 1: 19, 12, 9, 2 (total size: 42) + // group 2: 18, 13, 8, 3 (total size: 42) + // group 3: 17, 12, 7, 4 (total size: 42) + // group 4: 16, 11, 6, 5 (total size: 42) + List>> splits = ExportSnapshot.getBalancedSplits(files, 5); + assertEquals(5, splits.size()); + + String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"}; + verifyBalanceSplit(splits.get(0), split0, 42); + String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"}; + verifyBalanceSplit(splits.get(1), split1, 42); + String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"}; + verifyBalanceSplit(splits.get(2), split2, 42); + String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"}; + verifyBalanceSplit(splits.get(3), split3, 42); + String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"}; + verifyBalanceSplit(splits.get(4), split4, 42); + } + + private void verifyBalanceSplit(final List> split, + final String[] expected, final long expectedSize) { + assertEquals(expected.length, split.size()); + long totalSize = 0; + for (int i = 0; i < expected.length; ++i) { + Pair fileInfo = split.get(i); + assertEquals(expected[i], fileInfo.getFirst().getHfile()); + totalSize += fileInfo.getSecond(); + } + assertEquals(expectedSize, totalSize); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java new file mode 100644 index 00000000000..e2d7c110305 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotNoCluster.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.snapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestRule; + +/** + * Test Export Snapshot Tool + */ +@Category({MapReduceTests.class, MediumTests.class}) +public class TestExportSnapshotNoCluster { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). + withTimeout(this.getClass()).withLookingForStuckThread(true).build(); + private static final Log LOG = LogFactory.getLog(TestExportSnapshotNoCluster.class); + + protected final static HBaseCommonTestingUtility TEST_UTIL = new HBaseCommonTestingUtility(); + + private static FileSystem fs; + private static Path testDir; + + public static void setUpBaseConf(Configuration conf) { + conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); + conf.setInt("hbase.regionserver.msginterval", 100); + conf.setInt("hbase.client.pause", 250); + conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); + conf.setBoolean("hbase.master.enabletable.roundrobin", true); + conf.setInt("mapreduce.map.maxattempts", 10); + conf.set(HConstants.HBASE_DIR, testDir.toString()); + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + testDir = TEST_UTIL.getDataTestDir(); + fs = testDir.getFileSystem(TEST_UTIL.getConfiguration()); + + setUpBaseConf(TEST_UTIL.getConfiguration()); + } + + /** + * Mock a snapshot with files in the archive dir, + * two regions, and one reference file. + */ + @Test + public void testSnapshotWithRefsExportFileSystemState() throws Exception { + SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); + SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1", + "tableWithRefsV1"); + testSnapshotWithRefsExportFileSystemState(builder); + + snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, testDir); + builder = snapshotMock.createSnapshotV2("tableWithRefsV2", "tableWithRefsV2"); + testSnapshotWithRefsExportFileSystemState(builder); + } + + /** + * Generates a couple of regions for the specified SnapshotMock, + * and then it will run the export and verification. + */ + private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) + throws Exception { + Path[] r1Files = builder.addRegion(); + Path[] r2Files = builder.addRegion(); + builder.commit(); + int snapshotFilesCount = r1Files.length + r2Files.length; + + byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); + TableName tableName = builder.getTableDescriptor().getTableName(); + TestExportSnapshot.testExportFileSystemState(TEST_UTIL.getConfiguration(), + tableName, snapshotName, snapshotName, snapshotFilesCount, + testDir, getDestinationDir(), false, null); + } + + private Path getDestinationDir() { + Path path = new Path(new Path(testDir, "export-test"), "export-" + System.currentTimeMillis()); + LOG.info("HDFS export destination path: " + path); + return path; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java index d755961da6f..c375b0af16f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java @@ -54,7 +54,12 @@ public class TestMobExportSnapshot extends TestExportSnapshot { } @Override - protected boolean bypassRegion(HRegionInfo regionInfo) { - return MobUtils.isMobRegionInfo(regionInfo); + protected RegionPredicate getBypassRegionPredicate() { + return new RegionPredicate() { + @Override + public boolean evaluate(final HRegionInfo regionInfo) { + return MobUtils.isMobRegionInfo(regionInfo); + } + }; } }