HBASE-17995 improve log messages during snapshot tests.
Signed-off-by: Michael Stack <stack@apache.org>
This commit is contained in:
parent
d00850f66d
commit
8005d32943
|
@ -18,6 +18,8 @@
|
||||||
|
|
||||||
package org.apache.hadoop.hbase.mapreduce;
|
package org.apache.hadoop.hbase.mapreduce;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
@ -49,6 +51,7 @@ import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
|
||||||
public abstract class TableSnapshotInputFormatTestBase {
|
public abstract class TableSnapshotInputFormatTestBase {
|
||||||
|
private static final Log LOG = LogFactory.getLog(TableSnapshotInputFormatTestBase.class);
|
||||||
@Rule public final TestRule timeout = CategoryBasedTimeout.builder().
|
@Rule public final TestRule timeout = CategoryBasedTimeout.builder().
|
||||||
withTimeout(this.getClass()).withLookingForStuckThread(true).build();
|
withTimeout(this.getClass()).withLookingForStuckThread(true).build();
|
||||||
protected final HBaseTestingUtility UTIL = new HBaseTestingUtility();
|
protected final HBaseTestingUtility UTIL = new HBaseTestingUtility();
|
||||||
|
@ -191,11 +194,13 @@ public abstract class TableSnapshotInputFormatTestBase {
|
||||||
String snapshotName, byte[] startRow, byte[] endRow, int numRegions)
|
String snapshotName, byte[] startRow, byte[] endRow, int numRegions)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
try {
|
try {
|
||||||
|
LOG.debug("Ensuring table doesn't exist.");
|
||||||
util.deleteTable(tableName);
|
util.deleteTable(tableName);
|
||||||
} catch(Exception ex) {
|
} catch(Exception ex) {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG.info("creating table '" + tableName + "'");
|
||||||
if (numRegions > 1) {
|
if (numRegions > 1) {
|
||||||
util.createTable(tableName, FAMILIES, 1, startRow, endRow, numRegions);
|
util.createTable(tableName, FAMILIES, 1, startRow, endRow, numRegions);
|
||||||
} else {
|
} else {
|
||||||
|
@ -203,21 +208,22 @@ public abstract class TableSnapshotInputFormatTestBase {
|
||||||
}
|
}
|
||||||
Admin admin = util.getAdmin();
|
Admin admin = util.getAdmin();
|
||||||
|
|
||||||
// put some stuff in the table
|
LOG.info("put some stuff in the table");
|
||||||
Table table = util.getConnection().getTable(tableName);
|
Table table = util.getConnection().getTable(tableName);
|
||||||
util.loadTable(table, FAMILIES);
|
util.loadTable(table, FAMILIES);
|
||||||
|
|
||||||
Path rootDir = FSUtils.getRootDir(util.getConfiguration());
|
Path rootDir = FSUtils.getRootDir(util.getConfiguration());
|
||||||
FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
|
FileSystem fs = rootDir.getFileSystem(util.getConfiguration());
|
||||||
|
|
||||||
|
LOG.info("snapshot");
|
||||||
SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
|
SnapshotTestingUtils.createSnapshotAndValidate(admin, tableName,
|
||||||
Arrays.asList(FAMILIES), null, snapshotName, rootDir, fs, true);
|
Arrays.asList(FAMILIES), null, snapshotName, rootDir, fs, true);
|
||||||
|
|
||||||
// load different values
|
LOG.info("load different values");
|
||||||
byte[] value = Bytes.toBytes("after_snapshot_value");
|
byte[] value = Bytes.toBytes("after_snapshot_value");
|
||||||
util.loadTable(table, FAMILIES, value);
|
util.loadTable(table, FAMILIES, value);
|
||||||
|
|
||||||
// cause flush to create new files in the region
|
LOG.info("cause flush to create new files in the region");
|
||||||
admin.flush(tableName);
|
admin.flush(tableName);
|
||||||
table.close();
|
table.close();
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,6 +24,8 @@ import static org.mockito.Mockito.when;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.CategoryBasedTimeout;
|
import org.apache.hadoop.hbase.CategoryBasedTimeout;
|
||||||
|
@ -65,6 +67,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
|
||||||
|
|
||||||
@Category({VerySlowMapReduceTests.class, LargeTests.class})
|
@Category({VerySlowMapReduceTests.class, LargeTests.class})
|
||||||
public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBase {
|
public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBase {
|
||||||
|
private static final Log LOG = LogFactory.getLog(TestTableSnapshotInputFormat.class);
|
||||||
@Rule public final TestRule timeout = CategoryBasedTimeout.builder().
|
@Rule public final TestRule timeout = CategoryBasedTimeout.builder().
|
||||||
withTimeout(this.getClass()).withLookingForStuckThread(true).build();
|
withTimeout(this.getClass()).withLookingForStuckThread(true).build();
|
||||||
|
|
||||||
|
@ -343,10 +346,13 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
|
||||||
String snapshotName, byte[] startRow, byte[] endRow, Path tableDir, int numRegions,
|
String snapshotName, byte[] startRow, byte[] endRow, Path tableDir, int numRegions,
|
||||||
int expectedNumSplits, boolean shutdownCluster) throws Exception {
|
int expectedNumSplits, boolean shutdownCluster) throws Exception {
|
||||||
|
|
||||||
//create the table and snapshot
|
LOG.info("testing with MapReduce");
|
||||||
|
|
||||||
|
LOG.info("create the table and snapshot");
|
||||||
createTableAndSnapshot(util, tableName, snapshotName, startRow, endRow, numRegions);
|
createTableAndSnapshot(util, tableName, snapshotName, startRow, endRow, numRegions);
|
||||||
|
|
||||||
if (shutdownCluster) {
|
if (shutdownCluster) {
|
||||||
|
LOG.info("shutting down hbase cluster.");
|
||||||
util.shutdownMiniHBaseCluster();
|
util.shutdownMiniHBaseCluster();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -209,7 +209,7 @@ public final class SnapshotTestingUtils {
|
||||||
// check snapshot dir
|
// check snapshot dir
|
||||||
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
|
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
|
||||||
snapshotDescriptor, rootDir);
|
snapshotDescriptor, rootDir);
|
||||||
assertTrue(fs.exists(snapshotDir));
|
assertTrue("target snapshot directory, '"+ snapshotDir +"', doesn't exist.", fs.exists(snapshotDir));
|
||||||
|
|
||||||
SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
|
SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
|
||||||
|
|
||||||
|
@ -232,14 +232,14 @@ public final class SnapshotTestingUtils {
|
||||||
// Verify that there are store files in the specified families
|
// Verify that there are store files in the specified families
|
||||||
if (nonEmptyTestFamilies != null) {
|
if (nonEmptyTestFamilies != null) {
|
||||||
for (final byte[] familyName: nonEmptyTestFamilies) {
|
for (final byte[] familyName: nonEmptyTestFamilies) {
|
||||||
assertTrue(snapshotFamilies.contains(familyName));
|
assertTrue("Expected snapshot to contain family '" + Bytes.toString(familyName) + "', but it does not.", snapshotFamilies.contains(familyName));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify that there are no store files in the specified families
|
// Verify that there are no store files in the specified families
|
||||||
if (emptyTestFamilies != null) {
|
if (emptyTestFamilies != null) {
|
||||||
for (final byte[] familyName: emptyTestFamilies) {
|
for (final byte[] familyName: emptyTestFamilies) {
|
||||||
assertFalse(snapshotFamilies.contains(familyName));
|
assertFalse("Expected snapshot to skip empty family '" + Bytes.toString(familyName) + "', but it is present.", snapshotFamilies.contains(familyName));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -250,7 +250,7 @@ public final class SnapshotTestingUtils {
|
||||||
boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName)
|
boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName)
|
||||||
.getEncodedName());
|
.getEncodedName());
|
||||||
if (hasMob) {
|
if (hasMob) {
|
||||||
assertEquals(regions.size(), regionManifests.size() - 1);
|
assertEquals("Wrong number of regions.", regions.size(), regionManifests.size() - 1);
|
||||||
} else {
|
} else {
|
||||||
// if create snapshot when table splitting, parent region will be included to the snapshot
|
// if create snapshot when table splitting, parent region will be included to the snapshot
|
||||||
// region manifest. we should exclude the parent regions.
|
// region manifest. we should exclude the parent regions.
|
||||||
|
@ -262,13 +262,13 @@ public final class SnapshotTestingUtils {
|
||||||
}
|
}
|
||||||
regionCountExclusiveSplitParent++;
|
regionCountExclusiveSplitParent++;
|
||||||
}
|
}
|
||||||
assertEquals(regions.size(), regionCountExclusiveSplitParent);
|
assertEquals("Wrong number of regions.", regions.size(), regionCountExclusiveSplitParent);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify Regions (redundant check, see MasterSnapshotVerifier)
|
// Verify Regions (redundant check, see MasterSnapshotVerifier)
|
||||||
for (HRegionInfo info : regions) {
|
for (HRegionInfo info : regions) {
|
||||||
String regionName = info.getEncodedName();
|
String regionName = info.getEncodedName();
|
||||||
assertTrue(regionManifests.containsKey(regionName));
|
assertTrue("Missing region name: '" + regionName + "'", regionManifests.containsKey(regionName));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -403,20 +403,24 @@ public final class SnapshotTestingUtils {
|
||||||
throws Exception {
|
throws Exception {
|
||||||
if (!onlineSnapshot) {
|
if (!onlineSnapshot) {
|
||||||
try {
|
try {
|
||||||
|
LOG.info("prepping for offline snapshot.");
|
||||||
admin.disableTable(tableName);
|
admin.disableTable(tableName);
|
||||||
} catch (TableNotEnabledException tne) {
|
} catch (TableNotEnabledException tne) {
|
||||||
LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
|
LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
|
||||||
"already disabled.");
|
"already disabled.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
LOG.info("taking snapshot.");
|
||||||
admin.snapshot(snapshotNameString, tableName);
|
admin.snapshot(snapshotNameString, tableName);
|
||||||
|
|
||||||
|
LOG.info("Confirming snapshot exists.");
|
||||||
List<SnapshotDescription> snapshots =
|
List<SnapshotDescription> snapshots =
|
||||||
SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
|
SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
|
||||||
if (snapshots == null || snapshots.size() != 1) {
|
if (snapshots == null || snapshots.size() != 1) {
|
||||||
Assert.fail("Incorrect number of snapshots for table " + tableName);
|
Assert.fail("Incorrect number of snapshots for table " + tableName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LOG.info("validating snapshot.");
|
||||||
SnapshotTestingUtils.confirmSnapshotValid(
|
SnapshotTestingUtils.confirmSnapshotValid(
|
||||||
ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
|
ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
|
||||||
emptyFamilyNames, rootDir, admin, fs);
|
emptyFamilyNames, rootDir, admin, fs);
|
||||||
|
|
Loading…
Reference in New Issue