From 853a72bc465af5bff5b5fa18229a6e933ad7c52d Mon Sep 17 00:00:00 2001 From: stack Date: Wed, 15 Apr 2020 10:37:55 -0700 Subject: [PATCH] HBASE-24175 [Flakey Tests] TestSecureExportSnapshot FileNotFoundException Addendum: add test to check for '/tmp' references. --- .../snapshot/TestExportSnapshotAdjunct.java | 17 ++++++ .../hadoop/hbase/HBaseTestingUtility.java | 57 ++++++++++++------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java index eed10115e6b..cf8751ce062 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshotAdjunct.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.snapshot; +import static org.junit.Assert.assertFalse; +import java.util.Iterator; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -67,6 +70,20 @@ public class TestExportSnapshotAdjunct { TestExportSnapshot.setUpBaseConf(TEST_UTIL.getConfiguration()); TEST_UTIL.startMiniCluster(3); TEST_UTIL.startMiniMapReduceCluster(); + Configuration conf = TEST_UTIL.getConfiguration(); + for (Iterator> i = conf.iterator(); i.hasNext();) { + Map.Entry e = i.next(); + if (e.getValue().contains("java.io.tmpdir")) { + continue; + } + if (e.getValue().contains("hadoop.tmp.dir")) { + continue; + } + if (e.getValue().contains("hbase.tmp.dir")) { + continue; + } + assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("tmp")); + } } @AfterClass diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 4fd71a36894..7dda19e0d62 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -417,21 +417,6 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { createSubDir( "mapreduce.cluster.local.dir", testPath, "mapred-local-dir"); - - // Frustrate yarn's attempts at writing /tmp. - String property = "yarn.node-labels.fs-store.root-dir"; - createSubDir(property, testPath, property); - property = "yarn.nodemanager.log-dirs"; - createSubDir(property, testPath, property); - property = "yarn.nodemanager.remote-app-log-dir"; - createSubDir(property, testPath, property); - property = "yarn.timeline-service.entity-group-fs-store.active-dir"; - createSubDir(property, testPath, property); - property = "yarn.timeline-service.entity-group-fs-store.done-dir"; - createSubDir(property, testPath, property); - property = "yarn.nodemanager.remote-app-log-dir"; - createSubDir(property, testPath, property); - return testPath; } @@ -650,16 +635,24 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { return dfsCluster; } - /** This is used before starting HDFS and map-reduce mini-clusters */ + /** This is used before starting HDFS and map-reduce mini-clusters + * Run something like the below to check for the likes of '/tmp' references -- i.e. + * references outside of the test data dir -- in the conf. + * Configuration conf = TEST_UTIL.getConfiguration(); + * for (Iterator> i = conf.iterator(); i.hasNext();) { + * Map.Entry e = i.next(); + * assertFalse(e.getKey() + " " + e.getValue(), e.getValue().contains("/tmp")); + * } + */ private void createDirsAndSetProperties() throws IOException { setupClusterTestDir(); conf.set(TEST_DIRECTORY_KEY, clusterTestDir.getPath()); System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath()); - createDirAndSetProperty("cache_data", "test.cache.data"); - createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir"); - hadoopLogDir = createDirAndSetProperty("hadoop_logs", "hadoop.log.dir"); - createDirAndSetProperty("mapred_local", "mapreduce.cluster.local.dir"); - createDirAndSetProperty("mapred_temp", "mapreduce.cluster.temp.dir"); + createDirAndSetProperty("test.cache.data", "test.cache.data"); + createDirAndSetProperty("hadoop.tmp.dir", "hadoop.tmp.dir"); + hadoopLogDir = createDirAndSetProperty("hadoop.log.dir", "hadoop.log.dir"); + createDirAndSetProperty("mapreduce.cluster.local.dir", "mapreduce.cluster.local.dir"); + createDirAndSetProperty("mapreduce.cluster.temp.dir", "mapreduce.cluster.temp.dir"); enableShortCircuit(); Path root = getDataTestDirOnTestFS("hadoop"); @@ -671,6 +664,28 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { conf.set("mapreduce.job.working.dir", new Path(root, "mapred-working-dir").toString()); conf.set("yarn.app.mapreduce.am.staging-dir", new Path(root, "mapreduce-am-staging-root-dir").toString()); + + // Frustrate yarn's and hdfs's attempts at writing /tmp. + String property = "yarn.node-labels.fs-store.root-dir"; + createDirAndSetProperty(property, property); + property = "yarn.nodemanager.log-dirs"; + createDirAndSetProperty(property, property); + property = "yarn.nodemanager.remote-app-log-dir"; + createDirAndSetProperty(property, property); + property = "yarn.timeline-service.entity-group-fs-store.active-dir"; + createDirAndSetProperty(property, property); + property = "yarn.timeline-service.entity-group-fs-store.done-dir"; + createDirAndSetProperty(property, property); + property = "yarn.nodemanager.remote-app-log-dir"; + createDirAndSetProperty(property, property); + property = "dfs.journalnode.edits.dir"; + createDirAndSetProperty(property, property); + property = "dfs.datanode.shared.file.descriptor.paths"; + createDirAndSetProperty(property, property); + property = "nfs.dump.dir"; + createDirAndSetProperty(property, property); + property = "java.io.tmpdir"; + createDirAndSetProperty(property, property); } /**