From 4e7e48cdc1a5d3e6e9a4c5988c3db84b9be9e000 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 11 Jul 2016 18:06:24 -0700 Subject: [PATCH] HDFS-10300. TestDistCpSystem should share MiniDFSCluster. Contributed by John Zhuge. (cherry picked from commit f292624bd8dbdc1841f225a34346d0392fa76a47) (cherry picked from commit 38b1eafdbcbd91bd806b842729b84d84f3600524) --- .../apache/hadoop/tools/TestDistCpSystem.java | 185 ++++++++++-------- 1 file changed, 100 insertions(+), 85 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java index cd8656074c8..e3018a07730 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java @@ -18,7 +18,10 @@ package org.apache.hadoop.tools; +import static org.apache.hadoop.test.GenericTestUtils.getMethodName; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.io.OutputStream; @@ -26,26 +29,33 @@ import java.util.ArrayList; import java.util.List; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.util.ToolRunner; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; /** * A JUnit test for copying files recursively. */ -public class TestDistCpSystem extends TestCase { - +public class TestDistCpSystem { + @Rule + public Timeout globalTimeout = new Timeout(30000); + private static final String SRCDAT = "srcdat"; private static final String DSTDAT = "dstdat"; - + + private static MiniDFSCluster cluster; + private static Configuration conf; + private class FileEntry { String path; boolean isDir; @@ -92,60 +102,66 @@ private static FileStatus[] getFileStatus(FileSystem fs, private static void deldir(FileSystem fs, String topdir) throws IOException { fs.delete(new Path(topdir), true); } - - private void testPreserveUserHelper( - FileEntry[] srcEntries, - FileEntry[] dstEntries, - boolean createSrcDir, - boolean createTgtDir, - boolean update) throws Exception { - Configuration conf = null; - MiniDFSCluster cluster = null; - try { - final String testRoot = "/testdir"; - final String testSrcRel = SRCDAT; - final String testSrc = testRoot + "/" + testSrcRel; - final String testDstRel = DSTDAT; - final String testDst = testRoot + "/" + testDstRel; - conf = new Configuration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + private void testPreserveUserHelper(String testRoot, + FileEntry[] srcEntries, + FileEntry[] dstEntries, + boolean createSrcDir, + boolean createTgtDir, + boolean update) throws Exception { + final String testSrcRel = SRCDAT; + final String testSrc = testRoot + "/" + testSrcRel; + final String testDstRel = DSTDAT; + final String testDst = testRoot + "/" + testDstRel; - String nnUri = FileSystem.getDefaultUri(conf).toString(); - FileSystem fs = FileSystem.get(URI.create(nnUri), conf); - fs.mkdirs(new Path(testRoot)); - if (createSrcDir) { - fs.mkdirs(new Path(testSrc)); - } - if (createTgtDir) { - fs.mkdirs(new Path(testDst)); - } - - createFiles(fs, testRoot, srcEntries); - FileStatus[] srcstats = getFileStatus(fs, testRoot, srcEntries); - for(int i = 0; i < srcEntries.length; i++) { - fs.setOwner(srcstats[i].getPath(), "u" + i, null); - } - String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc, - nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst}; - - ToolRunner.run(conf, new DistCp(), args); - - String realTgtPath = testDst; - if (!createTgtDir) { - realTgtPath = testRoot; - } - FileStatus[] dststat = getFileStatus(fs, realTgtPath, dstEntries); - for(int i = 0; i < dststat.length; i++) { - assertEquals("i=" + i, "u" + i, dststat[i].getOwner()); - } - deldir(fs, testRoot); - } finally { - if (cluster != null) { cluster.shutdown(); } + String nnUri = FileSystem.getDefaultUri(conf).toString(); + FileSystem fs = FileSystem.get(URI.create(nnUri), conf); + fs.mkdirs(new Path(testRoot)); + if (createSrcDir) { + fs.mkdirs(new Path(testSrc)); + } + if (createTgtDir) { + fs.mkdirs(new Path(testDst)); + } + + createFiles(fs, testRoot, srcEntries); + FileStatus[] srcstats = getFileStatus(fs, testRoot, srcEntries); + for(int i = 0; i < srcEntries.length; i++) { + fs.setOwner(srcstats[i].getPath(), "u" + i, null); + } + String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc, + nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst}; + + ToolRunner.run(conf, new DistCp(), args); + + String realTgtPath = testDst; + if (!createTgtDir) { + realTgtPath = testRoot; + } + FileStatus[] dststat = getFileStatus(fs, realTgtPath, dstEntries); + for(int i = 0; i < dststat.length; i++) { + assertEquals("i=" + i, "u" + i, dststat[i].getOwner()); + } + deldir(fs, testRoot); + } + + @BeforeClass + public static void beforeClass() throws IOException { + conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitActive(); + } + + @AfterClass + public static void afterClass() throws IOException { + if (cluster != null) { + cluster.shutdown(); } } + @Test public void testPreserveUseNonEmptyDir() throws Exception { + String testRoot = "/testdir." + getMethodName(); FileEntry[] srcfiles = { new FileEntry(SRCDAT, true), new FileEntry(SRCDAT + "/a", false), @@ -160,12 +176,14 @@ public void testPreserveUseNonEmptyDir() throws Exception { new FileEntry(DSTDAT + "/b/c", false) }; - testPreserveUserHelper(srcfiles, srcfiles, false, true, false); - testPreserveUserHelper(srcfiles, dstfiles, false, false, false); + testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false); + testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } - - + + + @Test public void testPreserveUserEmptyDir() throws Exception { + String testRoot = "/testdir." + getMethodName(); FileEntry[] srcfiles = { new FileEntry(SRCDAT, true) }; @@ -174,22 +192,26 @@ public void testPreserveUserEmptyDir() throws Exception { new FileEntry(DSTDAT, true) }; - testPreserveUserHelper(srcfiles, srcfiles, false, true, false); - testPreserveUserHelper(srcfiles, dstfiles, false, false, false); + testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false); + testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } + @Test public void testPreserveUserSingleFile() throws Exception { + String testRoot = "/testdir." + getMethodName(); FileEntry[] srcfiles = { new FileEntry(SRCDAT, false) }; FileEntry[] dstfiles = { new FileEntry(DSTDAT, false) }; - testPreserveUserHelper(srcfiles, srcfiles, false, true, false); - testPreserveUserHelper(srcfiles, dstfiles, false, false, false); + testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false); + testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } - + + @Test public void testPreserveUserNonEmptyDirWithUpdate() throws Exception { + String testRoot = "/testdir." + getMethodName(); FileEntry[] srcfiles = { new FileEntry(SRCDAT + "/a", false), new FileEntry(SRCDAT + "/b", true), @@ -202,37 +224,30 @@ public void testPreserveUserNonEmptyDirWithUpdate() throws Exception { new FileEntry("b/c", false) }; - testPreserveUserHelper(srcfiles, dstfiles, true, true, true); + testPreserveUserHelper(testRoot, srcfiles, dstfiles, true, true, true); } @Test public void testSourceRoot() throws Exception { - MiniDFSCluster cluster = null; - Configuration conf = new Configuration(); - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitActive(); - FileSystem fs = cluster.getFileSystem(); + FileSystem fs = cluster.getFileSystem(); - String rootStr = fs.makeQualified(new Path("/")).toString(); + String rootStr = fs.makeQualified(new Path("/")).toString(); - // Case 1. The target does not exist. + String testRoot = "/testdir." + getMethodName(); - String tgtStr = fs.makeQualified(new Path("/nodir")).toString(); - String[] args = new String[]{ rootStr, tgtStr }; - Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0)); + // Case 1. The target does not exist. - // Case 2. The target exists. + Path tgtPath = new Path(testRoot + "/nodir"); + String tgtStr = fs.makeQualified(tgtPath).toString(); + String[] args = new String[]{rootStr, tgtStr}; + Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0)); - Path tgtPath2 = new Path("/dir"); - assertTrue(fs.mkdirs(tgtPath2)); - String tgtStr2 = fs.makeQualified(tgtPath2).toString(); - String[] args2 = new String[]{ rootStr, tgtStr2 }; - Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0)); - } finally { - if (cluster != null) { - cluster.shutdown(); - } - } + // Case 2. The target exists. + + Path tgtPath2 = new Path(testRoot + "/dir"); + assertTrue(fs.mkdirs(tgtPath2)); + String tgtStr2 = fs.makeQualified(tgtPath2).toString(); + String[] args2 = new String[]{rootStr, tgtStr2}; + Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0)); } } \ No newline at end of file