From d5aa4b9a9a91a880fdf3004a063f990bbd5f810d Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Tue, 1 May 2018 08:12:46 -0700 Subject: [PATCH] HDFS-13503. Fix TestFsck test failures on Windows. Contributed by Xiao Liang. (cherry picked from commit 9e2cfb2d3f1a18984d07c81f9c46626dd842402a) --- .../apache/hadoop/hdfs/MiniDFSCluster.java | 3 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 123 +++++++++++------- 2 files changed, 81 insertions(+), 45 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 0edc13ed3c4..6fa9a4f25a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -2894,7 +2894,8 @@ public class MiniDFSCluster implements AutoCloseable { * @return Storage directory */ public File getStorageDir(int dnIndex, int dirIndex) { - return new File(getBaseDirectory(), getStorageDirPath(dnIndex, dirIndex)); + return new File(determineDfsBaseDir(), + getStorageDirPath(dnIndex, dirIndex)); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 0c04738aa3f..bb27d048d9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.hdfs.MiniDFSCluster.HDFS_MINIDFS_BASEDIR; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -209,7 +210,9 @@ public class TestFsck { conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(4).build(); fs = cluster.getFileSystem(); final String fileName = "/srcdat"; util.createFiles(fs, fileName); @@ -297,7 +300,9 @@ public class TestFsck { setNumFiles(20).build(); FileSystem fs = null; conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(4).build(); fs = cluster.getFileSystem(); util.createFiles(fs, "/srcdat"); util.waitReplication(fs, "/srcdat", (short)3); @@ -315,7 +320,9 @@ public class TestFsck { conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); // Create a cluster with the current user, write some files - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(4).build(); final MiniDFSCluster c2 = cluster; final String dir = "/dfsck"; final Path dirpath = new Path(dir); @@ -361,8 +368,9 @@ public class TestFsck { DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, (5 * dfsBlockSize) + (dfsBlockSize - 1), 5 * dfsBlockSize); FileSystem fs = null; - cluster = new MiniDFSCluster.Builder(conf). - numDataNodes(numDatanodes).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(numDatanodes).build(); String topDir = "/srcdat"; fs = cluster.getFileSystem(); cluster.waitActive(); @@ -568,7 +576,9 @@ public class TestFsck { FileSystem fs = null; conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(4).build(); String topDir = "/srcdat"; fs = cluster.getFileSystem(); cluster.waitActive(); @@ -632,7 +642,9 @@ public class TestFsck { setNumFiles(4).build(); FileSystem fs = null; conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(4).build(); String topDir = "/srcdat"; String randomString = "HADOOP "; fs = cluster.getFileSystem(); @@ -685,7 +697,8 @@ public class TestFsck { final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits(); int blockSize = 2 * cellSize; conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes( + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes( numAllUnits + 1).build(); String topDir = "/myDir"; cluster.waitActive(); @@ -776,7 +789,9 @@ public class TestFsck { String outStr = null; short factor = 1; - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(1).build(); cluster.waitActive(); fs = cluster.getFileSystem(); Path file1 = new Path("/testCorruptBlock"); @@ -847,7 +862,9 @@ public class TestFsck { Random random = new Random(); String outStr = null; short factor = 1; - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(2).build(); cluster.waitActive(); fs = cluster.getFileSystem(); Path file1 = new Path("/testUnderMinReplicatedBlock"); @@ -919,9 +936,9 @@ public class TestFsck { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); DistributedFileSystem dfs; - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) - .racks(racks).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(numDn).hosts(hosts).racks(racks).build(); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); @@ -1068,7 +1085,8 @@ public class TestFsck { @Test public void testFsckError() throws Exception { // bring up a one-node cluster - cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); String fileName = "/test.txt"; Path filePath = new Path(fileName); FileSystem fs = cluster.getFileSystem(); @@ -1100,7 +1118,8 @@ public class TestFsck { conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); FileSystem fs = null; - cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); cluster.waitActive(); fs = cluster.getFileSystem(); DFSTestUtil util = new DFSTestUtil.Builder(). @@ -1163,7 +1182,8 @@ public class TestFsck { @Test public void testToCheckTheFsckCommandOnIllegalArguments() throws Exception { // bring up a one-node cluster - cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); String fileName = "/test.txt"; Path filePath = new Path(fileName); FileSystem fs = cluster.getFileSystem(); @@ -1207,8 +1227,9 @@ public class TestFsck { DistributedFileSystem dfs = null; // Startup a minicluster - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(numReplicas).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(numReplicas).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); @@ -1268,9 +1289,9 @@ public class TestFsck { DistributedFileSystem dfs = null; // Startup a minicluster - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) - .racks(racks).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = cluster.getFileSystem(); @@ -1377,7 +1398,9 @@ public class TestFsck { conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(4).build(); fs = cluster.getFileSystem(); final String fileName = "/srcdat"; util.createFiles(fs, fileName); @@ -1404,7 +1427,8 @@ public class TestFsck { */ @Test public void testFsckForSnapshotFiles() throws Exception { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes(1) .build(); String runFsck = runFsck(conf, 0, true, "/", "-includeSnapshots", "-files"); @@ -1439,9 +1463,9 @@ public class TestFsck { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); DistributedFileSystem dfs = null; - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) - .racks(racks).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); @@ -1494,9 +1518,9 @@ public class TestFsck { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2); DistributedFileSystem dfs; - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) - .racks(racks).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); @@ -1579,7 +1603,8 @@ public class TestFsck { replFactor); DistributedFileSystem dfs; - cluster = new MiniDFSCluster.Builder(conf) + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) .numDataNodes(numDn) .hosts(hosts) .racks(racks) @@ -1700,9 +1725,9 @@ public class TestFsck { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); DistributedFileSystem dfs = null; - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) - .racks(racks).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); @@ -1769,7 +1794,8 @@ public class TestFsck { */ @Test public void testStoragePoliciesCK() throws Exception { - cluster = new MiniDFSCluster.Builder(conf) + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) .numDataNodes(3) .storageTypes( new StorageType[] {StorageType.DISK, StorageType.ARCHIVE}) @@ -1812,9 +1838,9 @@ public class TestFsck { conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); DistributedFileSystem dfs; - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts) - .racks(racks).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(numDn).hosts(hosts).racks(racks).build(); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); @@ -1894,7 +1920,8 @@ public class TestFsck { replFactor); DistributedFileSystem dfs; - cluster = new MiniDFSCluster.Builder(conf) + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) .numDataNodes(numDn) .hosts(hosts) .racks(racks) @@ -2002,7 +2029,9 @@ public class TestFsck { int parityBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int totalSize = dataBlocks + parityBlocks; - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) + .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy( StripedFileTestUtil.getDefaultECPolicy().getName()); @@ -2069,7 +2098,8 @@ public class TestFsck { int numFiles = 3; int numSnapshots = 0; - cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); cluster.waitActive(); hdfs = cluster.getFileSystem(); DFSTestUtil util = new DFSTestUtil.Builder(). @@ -2149,7 +2179,8 @@ public class TestFsck { conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L); conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replication); - cluster = new MiniDFSCluster.Builder(conf).build(); + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build(); DistributedFileSystem dfs = cluster.getFileSystem(); cluster.waitActive(); @@ -2244,6 +2275,7 @@ public class TestFsck { HostsFileWriter hostsFileWriter = new HostsFileWriter(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replFactor); + conf.set(HDFS_MINIDFS_BASEDIR, GenericTestUtils.getRandomizedTempPath()); if (defineUpgradeDomain) { conf.setClass(DFSConfigKeys.DFS_NAMENODE_HOSTS_PROVIDER_CLASSNAME_KEY, CombinedHostFileManager.class, HostConfigManager.class); @@ -2295,7 +2327,8 @@ public class TestFsck { StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; - cluster = new MiniDFSCluster.Builder(conf) + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy( @@ -2366,7 +2399,8 @@ public class TestFsck { StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; - cluster = new MiniDFSCluster.Builder(conf) + File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath()); + cluster = new MiniDFSCluster.Builder(conf, builderBaseDir) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy( @@ -2427,7 +2461,8 @@ public class TestFsck { @Test(timeout = 300000) public void testFsckCorruptWhenOneReplicaIsCorrupt() throws Exception { - try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, + new File(GenericTestUtils.getRandomizedTempPath())) .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(2) .build()) { cluster.waitActive();