HDFS-10300. TestDistCpSystem should share MiniDFSCluster. Contributed by John Zhuge.
(cherry picked from commit f292624bd8)
This commit is contained in:
Andrew Wang
parent
be1a11c9c8
commit
38b1eafdbc
|
|
@ -18,7 +18,10 @@
|
|||
|
||||
package org.apache.hadoop.tools;
|
||||
|
||||
import static org.apache.hadoop.test.GenericTestUtils.getMethodName;
|
||||
import static org.hamcrest.core.Is.is;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
|
@ -26,26 +29,33 @@ import java.net.URI;
|
|||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.Timeout;
|
||||
|
||||
/**
|
||||
* A JUnit test for copying files recursively.
|
||||
*/
|
||||
|
||||
public class TestDistCpSystem extends TestCase {
|
||||
|
||||
public class TestDistCpSystem {
|
||||
@Rule
|
||||
public Timeout globalTimeout = new Timeout(30000);
|
||||
|
||||
private static final String SRCDAT = "srcdat";
|
||||
private static final String DSTDAT = "dstdat";
|
||||
|
||||
|
||||
private static MiniDFSCluster cluster;
|
||||
private static Configuration conf;
|
||||
|
||||
private class FileEntry {
|
||||
String path;
|
||||
boolean isDir;
|
||||
|
|
@ -92,60 +102,66 @@ public class TestDistCpSystem extends TestCase {
|
|||
private static void deldir(FileSystem fs, String topdir) throws IOException {
|
||||
fs.delete(new Path(topdir), true);
|
||||
}
|
||||
|
||||
private void testPreserveUserHelper(
|
||||
FileEntry[] srcEntries,
|
||||
FileEntry[] dstEntries,
|
||||
boolean createSrcDir,
|
||||
boolean createTgtDir,
|
||||
boolean update) throws Exception {
|
||||
Configuration conf = null;
|
||||
MiniDFSCluster cluster = null;
|
||||
try {
|
||||
final String testRoot = "/testdir";
|
||||
final String testSrcRel = SRCDAT;
|
||||
final String testSrc = testRoot + "/" + testSrcRel;
|
||||
final String testDstRel = DSTDAT;
|
||||
final String testDst = testRoot + "/" + testDstRel;
|
||||
|
||||
conf = new Configuration();
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
|
||||
private void testPreserveUserHelper(String testRoot,
|
||||
FileEntry[] srcEntries,
|
||||
FileEntry[] dstEntries,
|
||||
boolean createSrcDir,
|
||||
boolean createTgtDir,
|
||||
boolean update) throws Exception {
|
||||
final String testSrcRel = SRCDAT;
|
||||
final String testSrc = testRoot + "/" + testSrcRel;
|
||||
final String testDstRel = DSTDAT;
|
||||
final String testDst = testRoot + "/" + testDstRel;
|
||||
|
||||
String nnUri = FileSystem.getDefaultUri(conf).toString();
|
||||
FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
|
||||
fs.mkdirs(new Path(testRoot));
|
||||
if (createSrcDir) {
|
||||
fs.mkdirs(new Path(testSrc));
|
||||
}
|
||||
if (createTgtDir) {
|
||||
fs.mkdirs(new Path(testDst));
|
||||
}
|
||||
|
||||
createFiles(fs, testRoot, srcEntries);
|
||||
FileStatus[] srcstats = getFileStatus(fs, testRoot, srcEntries);
|
||||
for(int i = 0; i < srcEntries.length; i++) {
|
||||
fs.setOwner(srcstats[i].getPath(), "u" + i, null);
|
||||
}
|
||||
String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc,
|
||||
nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst};
|
||||
|
||||
ToolRunner.run(conf, new DistCp(), args);
|
||||
|
||||
String realTgtPath = testDst;
|
||||
if (!createTgtDir) {
|
||||
realTgtPath = testRoot;
|
||||
}
|
||||
FileStatus[] dststat = getFileStatus(fs, realTgtPath, dstEntries);
|
||||
for(int i = 0; i < dststat.length; i++) {
|
||||
assertEquals("i=" + i, "u" + i, dststat[i].getOwner());
|
||||
}
|
||||
deldir(fs, testRoot);
|
||||
} finally {
|
||||
if (cluster != null) { cluster.shutdown(); }
|
||||
String nnUri = FileSystem.getDefaultUri(conf).toString();
|
||||
FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
|
||||
fs.mkdirs(new Path(testRoot));
|
||||
if (createSrcDir) {
|
||||
fs.mkdirs(new Path(testSrc));
|
||||
}
|
||||
if (createTgtDir) {
|
||||
fs.mkdirs(new Path(testDst));
|
||||
}
|
||||
|
||||
createFiles(fs, testRoot, srcEntries);
|
||||
FileStatus[] srcstats = getFileStatus(fs, testRoot, srcEntries);
|
||||
for(int i = 0; i < srcEntries.length; i++) {
|
||||
fs.setOwner(srcstats[i].getPath(), "u" + i, null);
|
||||
}
|
||||
String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc,
|
||||
nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst};
|
||||
|
||||
ToolRunner.run(conf, new DistCp(), args);
|
||||
|
||||
String realTgtPath = testDst;
|
||||
if (!createTgtDir) {
|
||||
realTgtPath = testRoot;
|
||||
}
|
||||
FileStatus[] dststat = getFileStatus(fs, realTgtPath, dstEntries);
|
||||
for(int i = 0; i < dststat.length; i++) {
|
||||
assertEquals("i=" + i, "u" + i, dststat[i].getOwner());
|
||||
}
|
||||
deldir(fs, testRoot);
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws IOException {
|
||||
conf = new Configuration();
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
|
||||
cluster.waitActive();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClass() throws IOException {
|
||||
if (cluster != null) {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPreserveUseNonEmptyDir() throws Exception {
|
||||
String testRoot = "/testdir." + getMethodName();
|
||||
FileEntry[] srcfiles = {
|
||||
new FileEntry(SRCDAT, true),
|
||||
new FileEntry(SRCDAT + "/a", false),
|
||||
|
|
@ -160,12 +176,14 @@ public class TestDistCpSystem extends TestCase {
|
|||
new FileEntry(DSTDAT + "/b/c", false)
|
||||
};
|
||||
|
||||
testPreserveUserHelper(srcfiles, srcfiles, false, true, false);
|
||||
testPreserveUserHelper(srcfiles, dstfiles, false, false, false);
|
||||
testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false);
|
||||
testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void testPreserveUserEmptyDir() throws Exception {
|
||||
String testRoot = "/testdir." + getMethodName();
|
||||
FileEntry[] srcfiles = {
|
||||
new FileEntry(SRCDAT, true)
|
||||
};
|
||||
|
|
@ -174,22 +192,26 @@ public class TestDistCpSystem extends TestCase {
|
|||
new FileEntry(DSTDAT, true)
|
||||
};
|
||||
|
||||
testPreserveUserHelper(srcfiles, srcfiles, false, true, false);
|
||||
testPreserveUserHelper(srcfiles, dstfiles, false, false, false);
|
||||
testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false);
|
||||
testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPreserveUserSingleFile() throws Exception {
|
||||
String testRoot = "/testdir." + getMethodName();
|
||||
FileEntry[] srcfiles = {
|
||||
new FileEntry(SRCDAT, false)
|
||||
};
|
||||
FileEntry[] dstfiles = {
|
||||
new FileEntry(DSTDAT, false)
|
||||
};
|
||||
testPreserveUserHelper(srcfiles, srcfiles, false, true, false);
|
||||
testPreserveUserHelper(srcfiles, dstfiles, false, false, false);
|
||||
testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false);
|
||||
testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testPreserveUserNonEmptyDirWithUpdate() throws Exception {
|
||||
String testRoot = "/testdir." + getMethodName();
|
||||
FileEntry[] srcfiles = {
|
||||
new FileEntry(SRCDAT + "/a", false),
|
||||
new FileEntry(SRCDAT + "/b", true),
|
||||
|
|
@ -202,37 +224,30 @@ public class TestDistCpSystem extends TestCase {
|
|||
new FileEntry("b/c", false)
|
||||
};
|
||||
|
||||
testPreserveUserHelper(srcfiles, dstfiles, true, true, true);
|
||||
testPreserveUserHelper(testRoot, srcfiles, dstfiles, true, true, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSourceRoot() throws Exception {
|
||||
MiniDFSCluster cluster = null;
|
||||
Configuration conf = new Configuration();
|
||||
try {
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
|
||||
cluster.waitActive();
|
||||
FileSystem fs = cluster.getFileSystem();
|
||||
FileSystem fs = cluster.getFileSystem();
|
||||
|
||||
String rootStr = fs.makeQualified(new Path("/")).toString();
|
||||
String rootStr = fs.makeQualified(new Path("/")).toString();
|
||||
|
||||
// Case 1. The target does not exist.
|
||||
String testRoot = "/testdir." + getMethodName();
|
||||
|
||||
String tgtStr = fs.makeQualified(new Path("/nodir")).toString();
|
||||
String[] args = new String[]{ rootStr, tgtStr };
|
||||
Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0));
|
||||
// Case 1. The target does not exist.
|
||||
|
||||
// Case 2. The target exists.
|
||||
Path tgtPath = new Path(testRoot + "/nodir");
|
||||
String tgtStr = fs.makeQualified(tgtPath).toString();
|
||||
String[] args = new String[]{rootStr, tgtStr};
|
||||
Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0));
|
||||
|
||||
Path tgtPath2 = new Path("/dir");
|
||||
assertTrue(fs.mkdirs(tgtPath2));
|
||||
String tgtStr2 = fs.makeQualified(tgtPath2).toString();
|
||||
String[] args2 = new String[]{ rootStr, tgtStr2 };
|
||||
Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0));
|
||||
} finally {
|
||||
if (cluster != null) {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
// Case 2. The target exists.
|
||||
|
||||
Path tgtPath2 = new Path(testRoot + "/dir");
|
||||
assertTrue(fs.mkdirs(tgtPath2));
|
||||
String tgtStr2 = fs.makeQualified(tgtPath2).toString();
|
||||
String[] args2 = new String[]{rootStr, tgtStr2};
|
||||
Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue