Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23838 Adding debug logging to a few ExportSnapshot tests 

* hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
 Add logging of temp output dir and if tmp dir is being used at all.

* hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
 Don't create new Configurations. Use current. Set it into the launched
 ExportSnapshot too. Log as we make progress through stages so easier to
 find location of exception source.

* hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java
 Add makeQualified though looks to be redundant.

* hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestInfoServersACL.java
 Might fix a local fail where hbase doesn't seem to be up when we query
 jmx. Wait on meta being online.

M hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java
 Run all on the one cluster instance. See if helps w/ odd failure when
 run locally.
This commit is contained in:
stack 2020-02-13 13:38:42 -08:00
parent 488c2b2483
commit b7ef225609
6 changed files with 44 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
View File

@ -967,20 +967,19 @@ public class ExportSnapshot extends AbstractHBaseTool implements Tool {
Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX); Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf); FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf);
LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot);
Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf); FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf);
LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false) || boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false) ||
conf.get(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR) != null; conf.get(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR) != null;
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot, Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot,
destConf); destConf);
Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot); Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir; Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
LOG.debug("inputFs={}, inputRoot={}", inputFs.getUri().toString(), inputRoot);
LOG.debug("outputFs={}, outputRoot={}, outputFs.getUri().toString(), " +
"initialOutputSnapshotDir={}", outputRoot.toString(), skipTmp, initialOutputSnapshotDir);
// Find the necessary directory which need to change owner and group // Find the necessary directory which need to change owner and group
Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot); Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot);

27
hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
View File

@ -212,8 +212,8 @@ public class TestExportSnapshot {
final Path sourceDir, Path copyDir, final boolean overwrite, final Path sourceDir, Path copyDir, final boolean overwrite,
final RegionPredicate bypassregionPredicate, boolean success) throws Exception { final RegionPredicate bypassregionPredicate, boolean success) throws Exception {
URI hdfsUri = FileSystem.get(conf).getUri(); URI hdfsUri = FileSystem.get(conf).getUri();
FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); FileSystem fs = FileSystem.get(copyDir.toUri(), conf);
copyDir = copyDir.makeQualified(fs); copyDir = copyDir.makeQualified(hdfsUri, copyDir);
List<String> opts = new ArrayList<>(); List<String> opts = new ArrayList<>();
opts.add("--snapshot"); opts.add("--snapshot");
@ -227,30 +227,34 @@ public class TestExportSnapshot {
if (overwrite) opts.add("--overwrite"); if (overwrite) opts.add("--overwrite");
// Export Snapshot // Export Snapshot
int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()])); ExportSnapshot es = new ExportSnapshot();
es.setConf(conf);
int res = run(conf, es, opts.toArray(new String[opts.size()]));
assertEquals(success ? 0 : 1, res); assertEquals(success ? 0 : 1, res);
if (!success) { if (!success) {
final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName); final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
assertFalse(fs.exists(new Path(copyDir, targetDir))); assertFalse(copyDir.toString() + " " + targetDir.toString(),
fs.exists(new Path(copyDir, targetDir)));
return; return;
} }
LOG.info("Exported snapshot");
// Verify File-System state // Verify File-System state
FileStatus[] rootFiles = fs.listStatus(copyDir); FileStatus[] rootFiles = fs.listStatus(copyDir);
assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
for (FileStatus fileStatus: rootFiles) { for (FileStatus fileStatus: rootFiles) {
String name = fileStatus.getPath().getName(); String name = fileStatus.getPath().getName();
assertTrue(fileStatus.isDirectory()); assertTrue(fileStatus.toString(), fileStatus.isDirectory());
assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME) ||
name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY)); name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
} }
LOG.info("Verified filesystem state");
// compare the snapshot metadata and verify the hfiles // Compare the snapshot metadata and verify the hfiles
final FileSystem hdfs = FileSystem.get(hdfsUri, conf); final FileSystem hdfs = FileSystem.get(hdfsUri, conf);
final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName); final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName);
final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName); final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), fs, new Path(copyDir, targetDir));
fs, new Path(copyDir, targetDir));
Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName, Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName,
targetName, bypassregionPredicate); targetName, bypassregionPredicate);
assertEquals(filesExpected, snapshotFiles.size()); assertEquals(filesExpected, snapshotFiles.size());
@ -347,7 +351,8 @@ public class TestExportSnapshot {
private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir) private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
throws IOException { throws IOException {
Set<String> files = new HashSet<>(); Set<String> files = new HashSet<>();
int rootPrefix = root.makeQualified(fs).toString().length(); LOG.debug("List files in {} in root {} at {}", fs, root, dir);
int rootPrefix = root.makeQualified(fs.getUri(), root).toString().length();
FileStatus[] list = FSUtils.listStatus(fs, dir); FileStatus[] list = FSUtils.listStatus(fs, dir);
if (list != null) { if (list != null) {
for (FileStatus fstat: list) { for (FileStatus fstat: list) {

10
hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotV1NoCluster.java
View File

@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.snapshot;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -101,16 +102,17 @@ public class TestExportSnapshotV1NoCluster {
Path[] r2Files = builder.addRegion(); Path[] r2Files = builder.addRegion();
builder.commit(); builder.commit();
int snapshotFilesCount = r1Files.length + r2Files.length; int snapshotFilesCount = r1Files.length + r2Files.length;
String snapshotName = builder.getSnapshotDescription().getName(); String snapshotName = builder.getSnapshotDescription().getName();
TableName tableName = builder.getTableDescriptor().getTableName(); TableName tableName = builder.getTableDescriptor().getTableName();
TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(), TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(),
tableName, snapshotName, snapshotName, snapshotFilesCount, tableName, snapshotName, snapshotName, snapshotFilesCount,
testDir, getDestinationDir(testDir), false, null, true); testDir, getDestinationDir(testUtil, testDir), false, null, true);
} }
static Path getDestinationDir(Path testDir) { static Path getDestinationDir(HBaseCommonTestingUtility hctu, Path testDir) throws IOException {
Path path = new Path(new Path(testDir, "export-test"), "export-" + System.currentTimeMillis()); FileSystem fs = FileSystem.get(hctu.getConfiguration());
Path path = new Path(new Path(testDir, "export-test"),
"export-" + System.currentTimeMillis()).makeQualified(fs.getUri(), testDir);
LOG.info("HDFS export destination path: " + path); LOG.info("HDFS export destination path: " + path);
return path; return path;
} }

1
hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestInfoServersACL.java
View File

@ -138,6 +138,7 @@ public class TestInfoServersACL {
CLUSTER = new LocalHBaseCluster(conf, 1); CLUSTER = new LocalHBaseCluster(conf, 1);
CLUSTER.startup(); CLUSTER.startup();
CLUSTER.getActiveMaster().waitForMetaOnline();
} }
/** /**

39
hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStateStore.java
View File

@ -1,4 +1,4 @@
/** /*
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
@ -39,8 +38,8 @@ import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.junit.After; import org.junit.AfterClass;
import org.junit.Before; import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
@ -57,26 +56,25 @@ public class TestRegionStateStore {
private static final Logger LOG = LoggerFactory.getLogger(TestRegionStateStore.class); private static final Logger LOG = LoggerFactory.getLogger(TestRegionStateStore.class);
protected HBaseTestingUtility util; private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
@Before @BeforeClass
public void setup() throws Exception { public static void beforeClass() throws Exception {
util = new HBaseTestingUtility(); UTIL.startMiniCluster();
util.startMiniCluster();
} }
@After @AfterClass
public void tearDown() throws Exception { public static void tearDown() throws Exception {
util.shutdownMiniCluster(); UTIL.shutdownMiniCluster();
} }
@Test @Test
public void testVisitMetaForRegionExistingRegion() throws Exception { public void testVisitMetaForRegionExistingRegion() throws Exception {
final TableName tableName = TableName.valueOf("testVisitMetaForRegion"); final TableName tableName = TableName.valueOf("testVisitMetaForRegion");
util.createTable(tableName, "cf"); UTIL.createTable(tableName, "cf");
final List<HRegion> regions = util.getHBaseCluster().getRegions(tableName); final List<HRegion> regions = UTIL.getHBaseCluster().getRegions(tableName);
final String encodedName = regions.get(0).getRegionInfo().getEncodedName(); final String encodedName = regions.get(0).getRegionInfo().getEncodedName();
final RegionStateStore regionStateStore = util.getHBaseCluster().getMaster(). final RegionStateStore regionStateStore = UTIL.getHBaseCluster().getMaster().
getAssignmentManager().getRegionStateStore(); getAssignmentManager().getRegionStateStore();
final AtomicBoolean visitorCalled = new AtomicBoolean(false); final AtomicBoolean visitorCalled = new AtomicBoolean(false);
regionStateStore.visitMetaForRegion(encodedName, new RegionStateStore.RegionStateVisitor() { regionStateStore.visitMetaForRegion(encodedName, new RegionStateStore.RegionStateVisitor() {
@ -93,10 +91,10 @@ public class TestRegionStateStore {
@Test @Test
public void testVisitMetaForBadRegionState() throws Exception { public void testVisitMetaForBadRegionState() throws Exception {
final TableName tableName = TableName.valueOf("testVisitMetaForBadRegionState"); final TableName tableName = TableName.valueOf("testVisitMetaForBadRegionState");
util.createTable(tableName, "cf"); UTIL.createTable(tableName, "cf");
final List<HRegion> regions = util.getHBaseCluster().getRegions(tableName); final List<HRegion> regions = UTIL.getHBaseCluster().getRegions(tableName);
final String encodedName = regions.get(0).getRegionInfo().getEncodedName(); final String encodedName = regions.get(0).getRegionInfo().getEncodedName();
final RegionStateStore regionStateStore = util.getHBaseCluster().getMaster(). final RegionStateStore regionStateStore = UTIL.getHBaseCluster().getMaster().
getAssignmentManager().getRegionStateStore(); getAssignmentManager().getRegionStateStore();
// add the BAD_STATE which does not exist in enum RegionState.State // add the BAD_STATE which does not exist in enum RegionState.State
@ -105,7 +103,7 @@ public class TestRegionStateStore {
put.addColumn(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER, put.addColumn(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER,
Bytes.toBytes("BAD_STATE")); Bytes.toBytes("BAD_STATE"));
try (Table table = util.getConnection().getTable(TableName.META_TABLE_NAME)) { try (Table table = UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) {
table.put(put); table.put(put);
} }
@ -126,7 +124,7 @@ public class TestRegionStateStore {
@Test @Test
public void testVisitMetaForRegionNonExistingRegion() throws Exception { public void testVisitMetaForRegionNonExistingRegion() throws Exception {
final String encodedName = "fakeencodedregionname"; final String encodedName = "fakeencodedregionname";
final RegionStateStore regionStateStore = util.getHBaseCluster().getMaster(). final RegionStateStore regionStateStore = UTIL.getHBaseCluster().getMaster().
getAssignmentManager().getRegionStateStore(); getAssignmentManager().getRegionStateStore();
final AtomicBoolean visitorCalled = new AtomicBoolean(false); final AtomicBoolean visitorCalled = new AtomicBoolean(false);
regionStateStore.visitMetaForRegion(encodedName, new RegionStateStore.RegionStateVisitor() { regionStateStore.visitMetaForRegion(encodedName, new RegionStateStore.RegionStateVisitor() {
@ -138,5 +136,4 @@ public class TestRegionStateStore {
}); });
assertFalse("Visitor has been called, but it shouldn't.", visitorCalled.get()); assertFalse("Visitor has been called, but it shouldn't.", visitorCalled.get());
} }
} }

1
hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableProcedure.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.master.procedure; package org.apache.hadoop.hbase.master.procedure;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotDisabledException;