Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-24175 [Flakey Tests] TestSecureExportSnapshot FileNotFoundException 

Addendum: add test to check for '/tmp' references.
This commit is contained in:
stack 2020-04-15 10:37:55 -07:00
parent 88c02c12fa
commit 853a72bc46
2 changed files with 53 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java
View File

@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hbase.snapshot;
import static org.junit.Assert.assertFalse;
import java.util.Iterator;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -67,6 +70,20 @@ public class TestExportSnapshotAdjunct {
TestExportSnapshot.setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
Configuration conf = TEST_UTIL.getConfiguration();
for (Iterator<Map.Entry<String, String>> i = conf.iterator(); i.hasNext();) {
Map.Entry<String, String> e = i.next();
if (e.getValue().contains("java.io.tmpdir")) {
continue;
}
if (e.getValue().contains("hadoop.tmp.dir")) {
continue;
}
if (e.getValue().contains("hbase.tmp.dir")) {
continue;
}
assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("tmp"));
}
}
@AfterClass

57
hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
View File

@ -417,21 +417,6 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
createSubDir(
"mapreduce.cluster.local.dir",
testPath, "mapred-local-dir");
// Frustrate yarn's attempts at writing /tmp.
String property = "yarn.node-labels.fs-store.root-dir";
createSubDir(property, testPath, property);
property = "yarn.nodemanager.log-dirs";
createSubDir(property, testPath, property);
property = "yarn.nodemanager.remote-app-log-dir";
createSubDir(property, testPath, property);
property = "yarn.timeline-service.entity-group-fs-store.active-dir";
createSubDir(property, testPath, property);
property = "yarn.timeline-service.entity-group-fs-store.done-dir";
createSubDir(property, testPath, property);
property = "yarn.nodemanager.remote-app-log-dir";
createSubDir(property, testPath, property);
return testPath;
}
@ -650,16 +635,24 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
return dfsCluster;
}
/** This is used before starting HDFS and map-reduce mini-clusters */
/** This is used before starting HDFS and map-reduce mini-clusters
* Run something like the below to check for the likes of '/tmp' references -- i.e.
* references outside of the test data dir -- in the conf.
* Configuration conf = TEST_UTIL.getConfiguration();
* for (Iterator<Map.Entry<String, String>> i = conf.iterator(); i.hasNext();) {
* Map.Entry<String, String> e = i.next();
* assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("/tmp"));
* }
*/
private void createDirsAndSetProperties() throws IOException {
setupClusterTestDir();
conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
createDirAndSetProperty("cache_data", "test.cache.data");
createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir");
createDirAndSetProperty("mapred_local", "mapreduce.cluster.local.dir");
createDirAndSetProperty("mapred_temp", "mapreduce.cluster.temp.dir");
createDirAndSetProperty("test.cache.data", "test.cache.data");
createDirAndSetProperty("hadoop.tmp.dir", "hadoop.tmp.dir");
hadoopLogDir = createDirAndSetProperty("hadoop.log.dir", "hadoop.log.dir");
createDirAndSetProperty("mapreduce.cluster.local.dir", "mapreduce.cluster.local.dir");
createDirAndSetProperty("mapreduce.cluster.temp.dir", "mapreduce.cluster.temp.dir");
enableShortCircuit();
Path root = getDataTestDirOnTestFS("hadoop");
@ -671,6 +664,28 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
conf.set("mapreduce.job.working.dir", new Path(root, "mapred-working-dir").toString());
conf.set("yarn.app.mapreduce.am.staging-dir",
new Path(root, "mapreduce-am-staging-root-dir").toString());
// Frustrate yarn's and hdfs's attempts at writing /tmp.
String property = "yarn.node-labels.fs-store.root-dir";
createDirAndSetProperty(property, property);
property = "yarn.nodemanager.log-dirs";
createDirAndSetProperty(property, property);
property = "yarn.nodemanager.remote-app-log-dir";
createDirAndSetProperty(property, property);
property = "yarn.timeline-service.entity-group-fs-store.active-dir";
createDirAndSetProperty(property, property);
property = "yarn.timeline-service.entity-group-fs-store.done-dir";
createDirAndSetProperty(property, property);
property = "yarn.nodemanager.remote-app-log-dir";
createDirAndSetProperty(property, property);
property = "dfs.journalnode.edits.dir";
createDirAndSetProperty(property, property);
property = "dfs.datanode.shared.file.descriptor.paths";
createDirAndSetProperty(property, property);
property = "nfs.dump.dir";
createDirAndSetProperty(property, property);
property = "java.io.tmpdir";
createDirAndSetProperty(property, property);
}
/**