Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-9287. Parallel-testing hadoop-common. Contributed by Andrey Klochkov 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1488040 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jason Darrell Lowe 2013-05-30 22:43:16 +00:00
parent 131bfc91a6
commit 85623a2d75
61 changed files with 563 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -151,6 +151,8 @@ Trunk (Unreleased)
HADOOP-9540. Expose the InMemoryS3 and S3N FilesystemStores implementations
for Unit testing. (Hari via stevel)
HADOOP-9287. Parallel-testing hadoop-common (Andrey Klochkov via jlowe)
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.

17
hadoop-common-project/hadoop-common/pom.xml
View File

@ -751,6 +751,23 @@
</plugins>
</build>
</profile>
<profile>
<id>parallel-tests</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkMode>perthread</forkMode>
<threadCount>${testsThreadCount}</threadCount>
<parallel>classes</parallel>
<argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -DminiClusterDedicatedDirs=true</argLine>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -40,6 +40,7 @@ import static java.util.concurrent.TimeUnit.*;
import junit.framework.TestCase;
import static org.junit.Assert.assertArrayEquals;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.Path;
@ -50,8 +51,8 @@ import org.codehaus.jackson.map.ObjectMapper;
public class TestConfiguration extends TestCase {
private Configuration conf;
final static String CONFIG = new File("./test-config.xml").getAbsolutePath();
final static String CONFIG2 = new File("./test-config2.xml").getAbsolutePath();
final static String CONFIG = new File("./test-config-TestConfiguration.xml").getAbsolutePath();
final static String CONFIG2 = new File("./test-config2-TestConfiguration.xml").getAbsolutePath();
final static Random RAN = new Random();
final static String XMLHEADER =
IBM_JAVA?"<?xml version=\"1.0\" encoding=\"UTF-8\"?><configuration>":

8
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
View File

@ -36,11 +36,9 @@ import org.junit.Test;
public class TestConfigurationDeprecation {
private Configuration conf;
final static String CONFIG = new File("./test-config.xml").getAbsolutePath();
final static String CONFIG2 =
new File("./test-config2.xml").getAbsolutePath();
final static String CONFIG3 =
new File("./test-config3.xml").getAbsolutePath();
final static String CONFIG = new File("./test-config-TestConfigurationDeprecation.xml").getAbsolutePath();
final static String CONFIG2 = new File("./test-config2-TestConfigurationDeprecation.xml").getAbsolutePath();
final static String CONFIG3 = new File("./test-config3-TestConfigurationDeprecation.xml").getAbsolutePath();
BufferedWriter out;
static {

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
View File

@ -39,6 +39,8 @@ public abstract class FCStatisticsBaseTest {
static protected int blockSize = 512;
static protected int numBlocks = 1;
protected final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
//fc should be set appropriately by the deriving test.
protected static FileContext fc = null;
@ -47,7 +49,7 @@ public abstract class FCStatisticsBaseTest {
URI fsUri = getFsUri();
Statistics stats = FileContext.getStatistics(fsUri);
Assert.assertEquals(0, stats.getBytesRead());
Path filePath = getTestRootPath(fc, "file1");
Path filePath = fileContextTestHelper .getTestRootPath(fc, "file1");
createFile(fc, filePath, numBlocks, blockSize);
Assert.assertEquals(0, stats.getBytesRead());

79
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java
View File

@ -34,8 +34,6 @@ import org.junit.Before;
import org.junit.Test;
import org.mortbay.log.Log;
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
/**
* <p>
* A collection of tests for the {@link FileSystem}.
@ -55,7 +53,7 @@ import static org.apache.hadoop.fs.FileSystemTestHelper.*;
* @AfterClass public static void ClusterShutdownAtEnd()
* </p>
*/
public abstract class FSMainOperationsBaseTest {
public abstract class FSMainOperationsBaseTest extends FileSystemTestHelper {
private static String TEST_DIR_AAA2 = "test/hadoop2/aaa";
private static String TEST_DIR_AAA = "test/hadoop/aaa";
@ -63,8 +61,8 @@ public abstract class FSMainOperationsBaseTest {
private static String TEST_DIR_AXX = "test/hadoop/axx";
private static int numBlocks = 2;
protected static FileSystem fSys;
protected FileSystem fSys;
final private static PathFilter DEFAULT_FILTER = new PathFilter() {
@Override
@ -87,8 +85,18 @@ public abstract class FSMainOperationsBaseTest {
protected static final byte[] data = getFileData(numBlocks,
getDefaultBlockSize());
abstract protected FileSystem createFileSystem() throws Exception;
public FSMainOperationsBaseTest() {
}
public FSMainOperationsBaseTest(String testRootDir) {
super(testRootDir);
}
@Before
public void setUp() throws Exception {
fSys = createFileSystem();
fSys.mkdirs(getTestRootPath(fSys, "test"));
}
@ -157,7 +165,7 @@ public abstract class FSMainOperationsBaseTest {
// Now open a file relative to the wd we just set above.
Path absolutePath = new Path(absoluteDir, "foo");
FileSystemTestHelper.createFile(fSys, absolutePath);
createFile(fSys, absolutePath);
fSys.open(new Path("foo")).close();
@ -308,11 +316,11 @@ public abstract class FSMainOperationsBaseTest {
paths = fSys.listStatus(getTestRootPath(fSys, "test/hadoop"));
Assert.assertEquals(3, paths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/a"),
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, "test/hadoop/a"),
paths));
Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/b"),
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, "test/hadoop/b"),
paths));
Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/c"),
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, "test/hadoop/c"),
paths));
paths = fSys.listStatus(getTestRootPath(fSys, "test/hadoop/a"));
@ -359,9 +367,9 @@ public abstract class FSMainOperationsBaseTest {
FileStatus[] filteredPaths = fSys.listStatus(
getTestRootPath(fSys, "test/hadoop"), TEST_X_FILTER);
Assert.assertEquals(2,filteredPaths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AXA), filteredPaths));
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AXX), filteredPaths));
}
@ -417,9 +425,9 @@ public abstract class FSMainOperationsBaseTest {
FileStatus[] paths = fSys.globStatus(
getTestRootPath(fSys, "test/hadoop*"));
Assert.assertEquals(2, paths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
"test/hadoop"), paths));
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
"test/hadoop2"), paths));
}
@ -442,10 +450,10 @@ public abstract class FSMainOperationsBaseTest {
FileStatus[] paths = fSys.globStatus(
getTestRootPath(fSys, "test/hadoop*/*"));
Assert.assertEquals(4, paths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA), paths));
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA), paths));
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX), paths));
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA2), paths));
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AAA), paths));
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA), paths));
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX), paths));
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AAA2), paths));
}
@Test
@ -466,9 +474,9 @@ public abstract class FSMainOperationsBaseTest {
FileStatus[] paths = fSys.globStatus(
getTestRootPath(fSys, "test/hadoop/ax?"));
Assert.assertEquals(2, paths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AXA), paths));
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AXX), paths));
}
@ -513,11 +521,11 @@ public abstract class FSMainOperationsBaseTest {
getTestRootPath(fSys, "test/hadoop/*"),
DEFAULT_FILTER);
Assert.assertEquals(3, filteredPaths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AAA), filteredPaths));
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AXA), filteredPaths));
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AXX), filteredPaths));
}
@ -541,11 +549,11 @@ public abstract class FSMainOperationsBaseTest {
getTestRootPath(fSys, "test/hadoop/a??"),
DEFAULT_FILTER);
Assert.assertEquals(3, filteredPaths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA),
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AAA),
filteredPaths));
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA),
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA),
filteredPaths));
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX),
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX),
filteredPaths));
}
@ -569,9 +577,9 @@ public abstract class FSMainOperationsBaseTest {
getTestRootPath(fSys, "test/hadoop/*"),
TEST_X_FILTER);
Assert.assertEquals(2, filteredPaths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AXA), filteredPaths));
Assert.assertTrue(containsPath(getTestRootPath(fSys,
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys,
TEST_DIR_AXX), filteredPaths));
}
@ -617,9 +625,9 @@ public abstract class FSMainOperationsBaseTest {
getTestRootPath(fSys, "test/hadoop/a??"),
TEST_X_FILTER);
Assert.assertEquals(2, filteredPaths.length);
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA),
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA),
filteredPaths));
Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX),
Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX),
filteredPaths));
}
@ -1090,9 +1098,9 @@ public abstract class FSMainOperationsBaseTest {
public void testCopyToLocalWithUseRawLocalFileSystemOption() throws Exception {
Configuration conf = new Configuration();
FileSystem fSys = new RawLocalFileSystem();
Path fileToFS = new Path(TEST_ROOT_DIR, "fs.txt");
Path fileToLFS = new Path(TEST_ROOT_DIR, "test.txt");
Path crcFileAtLFS = new Path(TEST_ROOT_DIR, ".test.txt.crc");
Path fileToFS = new Path(getTestRootDir(), "fs.txt");
Path fileToLFS = new Path(getTestRootDir(), "test.txt");
Path crcFileAtLFS = new Path(getTestRootDir(), ".test.txt.crc");
fSys.initialize(new URI("file:///"), conf);
writeFile(fSys, fileToFS);
if (fSys.exists(crcFileAtLFS))
@ -1112,7 +1120,7 @@ public abstract class FSMainOperationsBaseTest {
}
protected void createFile(Path path) throws IOException {
FileSystemTestHelper.createFile(fSys, path);
createFile(fSys, path);
}
@SuppressWarnings("deprecation")
@ -1125,10 +1133,11 @@ public abstract class FSMainOperationsBaseTest {
Assert.assertEquals("Source exists", srcExists, exists(fSys, src));
Assert.assertEquals("Destination exists", dstExists, exists(fSys, dst));
}
private boolean containsPath(Path path, FileStatus[] filteredPaths)
private boolean containsTestRootPath(Path path, FileStatus[] filteredPaths)
throws IOException {
Path testRootPath = getTestRootPath(fSys, path.toString());
for(int i = 0; i < filteredPaths.length; i ++) {
if (getTestRootPath(fSys, path.toString()).equals(
if (testRootPath.equals(
filteredPaths[i].getPath()))
return true;
}

19
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
View File

@ -50,7 +50,8 @@ import org.apache.commons.logging.impl.Log4JLogger;
*/
public abstract class FileContextCreateMkdirBaseTest {
protected final FileContextTestHelper fileContextTestHelper;
protected static FileContext fc;
{
@ -63,6 +64,13 @@ public abstract class FileContextCreateMkdirBaseTest {
}
}
public FileContextCreateMkdirBaseTest() {
fileContextTestHelper = createFileContextHelper();
}
protected FileContextTestHelper createFileContextHelper() {
return new FileContextTestHelper();
}
@Before
public void setUp() throws Exception {
@ -149,4 +157,13 @@ public abstract class FileContextCreateMkdirBaseTest {
createFile(fc, f);
Assert.assertTrue(isFile(fc, f));
}
private Path getTestRootPath(FileContext fc) {
return fileContextTestHelper.getTestRootPath(fc);
}
private Path getTestRootPath(FileContext fc, String pathString) {
return fileContextTestHelper.getTestRootPath(fc, pathString);
}
}

38
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
View File

@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.EnumSet;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename;
@ -61,17 +62,10 @@ public abstract class FileContextMainOperationsBaseTest {
private static String TEST_DIR_AXA = "test/hadoop/axa";
private static String TEST_DIR_AXX = "test/hadoop/axx";
private static int numBlocks = 2;
public static final Path LOCAL_FS_ROOT_PATH;
static {
File testBuildData = new File(System.getProperty("test.build.data",
"build/test/data"));
Path localFsRootPath = new Path(testBuildData.getAbsolutePath(),
"root-uri");
LOCAL_FS_ROOT_PATH = localFsRootPath.makeQualified(LocalFileSystem.NAME, null);
}
public Path localFsRootPath;
protected final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
protected static FileContext fc;
final private static PathFilter DEFAULT_FILTER = new PathFilter() {
@ -97,13 +91,18 @@ public abstract class FileContextMainOperationsBaseTest {
@Before
public void setUp() throws Exception {
File testBuildData = new File(System.getProperty("test.build.data",
"build/test/data"), RandomStringUtils.randomAlphanumeric(10));
Path rootPath = new Path(testBuildData.getAbsolutePath(),
"root-uri");
localFsRootPath = rootPath.makeQualified(LocalFileSystem.NAME, null);
fc.mkdir(getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true);
}
@After
public void tearDown() throws Exception {
fc.delete(new Path(getAbsoluteTestRootPath(fc), new Path("test")), true);
fc.delete(LOCAL_FS_ROOT_PATH, true);
fc.delete(new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")), true);
fc.delete(localFsRootPath, true);
}
@ -136,7 +135,7 @@ public abstract class FileContextMainOperationsBaseTest {
public void testWorkingDirectory() throws Exception {
// First we cd to our test root
Path workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test"));
Path workDir = new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test"));
fc.setWorkingDirectory(workDir);
Assert.assertEquals(workDir, fc.getWorkingDirectory());
@ -149,7 +148,7 @@ public abstract class FileContextMainOperationsBaseTest {
// cd using a relative path
// Go back to our test root
workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test"));
workDir = new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test"));
fc.setWorkingDirectory(workDir);
Assert.assertEquals(workDir, fc.getWorkingDirectory());
@ -184,7 +183,7 @@ public abstract class FileContextMainOperationsBaseTest {
// Try a URI
absoluteDir = new Path(LOCAL_FS_ROOT_PATH, "existingDir");
absoluteDir = new Path(localFsRootPath, "existingDir");
fc.mkdir(absoluteDir, FileContext.DEFAULT_PERM, true);
fc.setWorkingDirectory(absoluteDir);
Assert.assertEquals(absoluteDir, fc.getWorkingDirectory());
@ -1181,6 +1180,7 @@ public abstract class FileContextMainOperationsBaseTest {
Assert.assertEquals("Source exists", srcExists, exists(fc, src));
Assert.assertEquals("Destination exists", dstExists, exists(fc, dst));
}
private boolean containsPath(Path path, FileStatus[] filteredPaths)
throws IOException {
for(int i = 0; i < filteredPaths.length; i ++) {
@ -1189,5 +1189,9 @@ public abstract class FileContextMainOperationsBaseTest {
return true;
}
return false;
}
}
private Path getTestRootPath(FileContext fc, String pathString) {
return fileContextTestHelper.getTestRootPath(fc, pathString);
}
}

23
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
View File

@ -68,16 +68,25 @@ public abstract class FileContextPermissionBase {
}
}
protected static FileContext fc;
protected FileContextTestHelper fileContextTestHelper;
protected FileContext fc;
protected FileContextTestHelper getFileContextHelper() {
return new FileContextTestHelper();
}
protected abstract FileContext getFileContext() throws Exception;
@Before
public void setUp() throws Exception {
fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
fileContextTestHelper = getFileContextHelper();
fc = getFileContext();
fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
}
@After
public void tearDown() throws Exception {
fc.delete(getTestRootPath(fc), true);
fc.delete(fileContextTestHelper.getTestRootPath(fc), true);
}
private void cleanupFile(FileContext fc, Path name) throws IOException {
@ -93,8 +102,8 @@ public abstract class FileContextPermissionBase {
return;
}
String filename = "foo";
Path f = getTestRootPath(fc, filename);
createFile(fc, filename);
Path f = fileContextTestHelper.getTestRootPath(fc, filename);
fileContextTestHelper.createFile(fc, filename);
doFilePermissionCheck(FileContext.FILE_DEFAULT_PERM.applyUMask(fc.getUMask()),
fc.getFileStatus(f).getPermission());
}
@ -108,7 +117,7 @@ public abstract class FileContextPermissionBase {
}
String filename = "foo";
Path f = getTestRootPath(fc, filename);
Path f = fileContextTestHelper.getTestRootPath(fc, filename);
createFile(fc, f);
try {
@ -133,7 +142,7 @@ public abstract class FileContextPermissionBase {
}
String filename = "bar";
Path f = getTestRootPath(fc, filename);
Path f = fileContextTestHelper.getTestRootPath(fc, filename);
createFile(fc, f);
List<String> groups = null;
try {

1
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java
View File

@ -43,6 +43,7 @@ public abstract class FileContextSymlinkBaseTest {
static final int blockSize = 8192;
static final int fileSize = 16384;
protected final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
protected static FileContext fc;
abstract protected String getScheme();

44
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.FileNotFoundException;
import java.util.EnumSet;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.CreateOpts.BlockSize;
import org.apache.hadoop.io.IOUtils;
@ -31,15 +32,18 @@ import org.junit.Assert;
* Helper class for unit tests.
*/
public final class FileContextTestHelper {
// The test root is relative to the <wd>/build/test/data by default
public static String TEST_ROOT_DIR =
System.getProperty("test.build.data", "build/test/data") + "/test";
private static final int DEFAULT_BLOCK_SIZE = 1024;
private static final int DEFAULT_NUM_BLOCKS = 2;
private static String absTestRootDir = null;
/** Hidden constructor */
private FileContextTestHelper() {}
private final String testRootDir;
private String absTestRootDir = null;
/**
* Create a context with test root relative to the <wd>/build/test/data
*/
public FileContextTestHelper() {
testRootDir = System.getProperty("test.build.data", "target/test/data") + "/" + RandomStringUtils.randomAlphanumeric(10);
}
public static int getDefaultBlockSize() {
return DEFAULT_BLOCK_SIZE;
@ -53,36 +57,36 @@ public final class FileContextTestHelper {
return data;
}
public static Path getTestRootPath(FileContext fc) {
return fc.makeQualified(new Path(TEST_ROOT_DIR));
public Path getTestRootPath(FileContext fc) {
return fc.makeQualified(new Path(testRootDir));
}
public static Path getTestRootPath(FileContext fc, String pathString) {
return fc.makeQualified(new Path(TEST_ROOT_DIR, pathString));
public Path getTestRootPath(FileContext fc, String pathString) {
return fc.makeQualified(new Path(testRootDir, pathString));
}
// the getAbsolutexxx method is needed because the root test dir
// can be messed up by changing the working dir.
public static String getAbsoluteTestRootDir(FileContext fc)
public String getAbsoluteTestRootDir(FileContext fc)
throws IOException {
if (absTestRootDir == null) {
if (new Path(TEST_ROOT_DIR).isAbsolute()) {
absTestRootDir = TEST_ROOT_DIR;
if (new Path(testRootDir).isAbsolute()) {
absTestRootDir = testRootDir;
} else {
absTestRootDir = fc.getWorkingDirectory().toString() + "/"
+ TEST_ROOT_DIR;
+ testRootDir;
}
}
return absTestRootDir;
}
public static Path getAbsoluteTestRootPath(FileContext fc) throws IOException {
public Path getAbsoluteTestRootPath(FileContext fc) throws IOException {
return fc.makeQualified(new Path(getAbsoluteTestRootDir(fc)));
}
public static Path getDefaultWorkingDirectory(FileContext fc)
public Path getDefaultWorkingDirectory(FileContext fc)
throws IOException {
return getTestRootPath(fc, "/user/" + System.getProperty("user.name"))
.makeQualified(fc.getDefaultFileSystem().getUri(),
@ -116,12 +120,12 @@ public final class FileContextTestHelper {
return createFile(fc, path, DEFAULT_NUM_BLOCKS, CreateOpts.createParent());
}
public static long createFile(FileContext fc, String name) throws IOException {
public long createFile(FileContext fc, String name) throws IOException {
Path path = getTestRootPath(fc, name);
return createFile(fc, path);
}
public static long createFileNonRecursive(FileContext fc, String name)
public long createFileNonRecursive(FileContext fc, String name)
throws IOException {
Path path = getTestRootPath(fc, name);
return createFileNonRecursive(fc, path);
@ -190,7 +194,7 @@ public final class FileContextTestHelper {
return buffer;
}
public static FileStatus containsPath(FileContext fc, Path path,
public FileStatus containsPath(FileContext fc, Path path,
FileStatus[] dirList)
throws IOException {
return containsPath(getTestRootPath(fc, path.toString()), dirList);
@ -206,7 +210,7 @@ public final class FileContextTestHelper {
return null;
}
public static FileStatus containsPath(FileContext fc, String path,
public FileStatus containsPath(FileContext fc, String path,
FileStatus[] dirList)
throws IOException {
return containsPath(fc, new Path(path), dirList);

14
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.hadoop.fs;
import static org.apache.hadoop.fs.FileContextTestHelper.getTestRootPath;
import static org.apache.hadoop.fs.FileContextTestHelper.readFile;
import static org.apache.hadoop.fs.FileContextTestHelper.writeFile;
import static org.junit.Assert.assertTrue;
@ -44,6 +43,7 @@ import org.junit.Test;
* </p>
*/
public abstract class FileContextUtilBase {
protected final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
protected FileContext fc;
{
@ -58,19 +58,19 @@ public abstract class FileContextUtilBase {
@Before
public void setUp() throws Exception {
fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
}
@After
public void tearDown() throws Exception {
fc.delete(getTestRootPath(fc), true);
fc.delete(fileContextTestHelper.getTestRootPath(fc), true);
}
@Test
public void testFcCopy() throws Exception{
final String ts = "some random text";
Path file1 = getTestRootPath(fc, "file1");
Path file2 = getTestRootPath(fc, "file2");
Path file1 = fileContextTestHelper.getTestRootPath(fc, "file1");
Path file2 = fileContextTestHelper.getTestRootPath(fc, "file2");
writeFile(fc, file1, ts.getBytes());
assertTrue(fc.util().exists(file1));
@ -87,8 +87,8 @@ public abstract class FileContextUtilBase {
public void testRecursiveFcCopy() throws Exception {
final String ts = "some random text";
Path dir1 = getTestRootPath(fc, "dir1");
Path dir2 = getTestRootPath(fc, "dir2");
Path dir1 = fileContextTestHelper.getTestRootPath(fc, "dir1");
Path dir2 = fileContextTestHelper.getTestRootPath(fc, "dir2");
Path file1 = new Path(dir1, "file1");
fc.mkdir(dir1, null, false);

57
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java
View File

@ -23,27 +23,39 @@ import java.net.URI;
import java.util.Random;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
/**
* Helper class for unit tests.
*/
public final class FileSystemTestHelper {
// The test root is relative to the <wd>/build/test/data by default
public static String TEST_ROOT_DIR =
System.getProperty("test.build.data", "target/test/data") + "/test";
public class FileSystemTestHelper {
private static final int DEFAULT_BLOCK_SIZE = 1024;
private static final int DEFAULT_NUM_BLOCKS = 2;
private static final short DEFAULT_NUM_REPL = 1;
private static String absTestRootDir = null;
/** Hidden constructor */
private FileSystemTestHelper() {}
protected final String testRootDir;
private String absTestRootDir = null;
/**
* Create helper with test root located at <wd>/build/test/data
*/
public FileSystemTestHelper() {
this(System.getProperty("test.build.data", "target/test/data") + "/" + RandomStringUtils.randomAlphanumeric(10));
}
/**
* Create helper with the specified test root dir
*/
public FileSystemTestHelper(String testRootDir) {
this.testRootDir = testRootDir;
}
public static void addFileSystemForTesting(URI uri, Configuration conf,
FileSystem fs) throws IOException {
FileSystem.addFileSystemForTesting(uri, conf, fs);
@ -61,19 +73,22 @@ public final class FileSystemTestHelper {
return data;
}
public String getTestRootDir() {
return testRootDir;
}
/*
* get testRootPath qualified for fSys
*/
public static Path getTestRootPath(FileSystem fSys) {
return fSys.makeQualified(new Path(TEST_ROOT_DIR));
public Path getTestRootPath(FileSystem fSys) {
return fSys.makeQualified(new Path(testRootDir));
}
/*
* get testRootPath + pathString qualified for fSys
*/
public static Path getTestRootPath(FileSystem fSys, String pathString) {
return fSys.makeQualified(new Path(TEST_ROOT_DIR, pathString));
public Path getTestRootPath(FileSystem fSys, String pathString) {
return fSys.makeQualified(new Path(testRootDir, pathString));
}
@ -82,25 +97,25 @@ public final class FileSystemTestHelper {
// is often relative to the working directory of process
// running the unit tests.
static String getAbsoluteTestRootDir(FileSystem fSys)
String getAbsoluteTestRootDir(FileSystem fSys)
throws IOException {
// NOTE: can't cache because of different filesystems!
//if (absTestRootDir == null)
if (new Path(TEST_ROOT_DIR).isAbsolute()) {
absTestRootDir = TEST_ROOT_DIR;
if (new Path(testRootDir).isAbsolute()) {
absTestRootDir = testRootDir;
} else {
absTestRootDir = fSys.getWorkingDirectory().toString() + "/"
+ TEST_ROOT_DIR;
+ testRootDir;
}
//}
return absTestRootDir;
}
public static Path getAbsoluteTestRootPath(FileSystem fSys) throws IOException {
public Path getAbsoluteTestRootPath(FileSystem fSys) throws IOException {
return fSys.makeQualified(new Path(getAbsoluteTestRootDir(fSys)));
}
public static Path getDefaultWorkingDirectory(FileSystem fSys)
public Path getDefaultWorkingDirectory(FileSystem fSys)
throws IOException {
return getTestRootPath(fSys, "/user/" + System.getProperty("user.name"))
.makeQualified(fSys.getUri(),
@ -136,7 +151,7 @@ public final class FileSystemTestHelper {
return createFile(fSys, path, DEFAULT_NUM_BLOCKS, DEFAULT_BLOCK_SIZE, DEFAULT_NUM_REPL, true);
}
public static long createFile(FileSystem fSys, String name) throws IOException {
public long createFile(FileSystem fSys, String name) throws IOException {
Path path = getTestRootPath(fSys, name);
return createFile(fSys, path);
}
@ -188,7 +203,7 @@ public final class FileSystemTestHelper {
return s;
}
public static FileStatus containsPath(FileSystem fSys, Path path,
public FileStatus containsPath(FileSystem fSys, Path path,
FileStatus[] dirList)
throws IOException {
for(int i = 0; i < dirList.length; i ++) {
@ -210,7 +225,7 @@ public final class FileSystemTestHelper {
}
public static FileStatus containsPath(FileSystem fSys, String path, FileStatus[] dirList)
public FileStatus containsPath(FileSystem fSys, String path, FileStatus[] dirList)
throws IOException {
return containsPath(fSys, new Path(path), dirList);
}

9
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java
View File

@ -28,10 +28,14 @@ import org.junit.Test;
public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTest {
@Override
protected FileSystem createFileSystem() throws IOException {
return FileSystem.getLocal(new Configuration());
}
@Override
@Before
public void setUp() throws Exception {
fSys = FileSystem.getLocal(new Configuration());
super.setUp();
}
@ -52,8 +56,7 @@ public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTes
@Test
@Override
public void testWDAbsolute() throws IOException {
Path absoluteDir = FileSystemTestHelper.getTestRootPath(fSys,
"test/existingDir");
Path absoluteDir = getTestRootPath(fSys, "test/existingDir");
fSys.mkdirs(absoluteDir);
fSys.setWorkingDirectory(absoluteDir);
Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());

6
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFcLocalFsPermission.java
View File

@ -29,7 +29,6 @@ public class TestFcLocalFsPermission extends
@Override
@Before
public void setUp() throws Exception {
fc = FileContext.getLocalFSFileContext();
super.setUp();
}
@ -39,4 +38,9 @@ public class TestFcLocalFsPermission extends
super.tearDown();
}
@Override
protected FileContext getFileContext() throws UnsupportedFileSystemException {
return FileContext.getLocalFSFileContext();
}
}

9
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextDeleteOnExit.java
View File

@ -35,6 +35,7 @@ public class TestFileContextDeleteOnExit {
private static int blockSize = 1024;
private static int numBlocks = 2;
private final FileContextTestHelper helper = new FileContextTestHelper();
private FileContext fc;
@Before
@ -44,7 +45,7 @@ public class TestFileContextDeleteOnExit {
@After
public void tearDown() throws IOException {
fc.delete(getTestRootPath(fc), true);
fc.delete(helper.getTestRootPath(fc), true);
}
@ -60,7 +61,7 @@ public class TestFileContextDeleteOnExit {
@Test
public void testDeleteOnExit() throws Exception {
// Create deleteOnExit entries
Path file1 = getTestRootPath(fc, "file1");
Path file1 = helper.getTestRootPath(fc, "file1");
createFile(fc, file1, numBlocks, blockSize);
fc.deleteOnExit(file1);
checkDeleteOnExitData(1, fc, file1);
@ -68,12 +69,12 @@ public class TestFileContextDeleteOnExit {
// Ensure shutdown hook is added
Assert.assertTrue(ShutdownHookManager.get().hasShutdownHook(FileContext.FINALIZER));
Path file2 = getTestRootPath(fc, "dir1/file2");
Path file2 = helper.getTestRootPath(fc, "dir1/file2");
createFile(fc, file2, numBlocks, blockSize);
fc.deleteOnExit(file2);
checkDeleteOnExitData(1, fc, file1, file2);
Path dir = getTestRootPath(fc, "dir3/dir4/dir5/dir6");
Path dir = helper.getTestRootPath(fc, "dir3/dir4/dir5/dir6");
createFile(fc, dir, numBlocks, blockSize);
fc.deleteOnExit(dir);
checkDeleteOnExitData(1, fc, file1, file2, dir);

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
View File

@ -52,7 +52,7 @@ public class TestLocalFSFileContextMainOperations extends FileContextMainOperati
@Test
public void testDefaultFilePermission() throws IOException {
Path file = FileContextTestHelper.getTestRootPath(fc,
Path file = fileContextTestHelper.getTestRootPath(fc,
"testDefaultFilePermission");
FileContextTestHelper.createFile(fc, file);
FsPermission expect = FileContext.FILE_DEFAULT_PERM.applyUMask(fc.getUMask());

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextSymlink.java
View File

@ -25,7 +25,6 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileUtil;
import static org.apache.hadoop.fs.FileContextTestHelper.*;
import static org.junit.Assert.*;
import static org.junit.Assume.assumeTrue;
import org.junit.Test;
@ -43,12 +42,12 @@ public class TestLocalFSFileContextSymlink extends FileContextSymlinkBaseTest {
@Override
protected String testBaseDir1() throws IOException {
return getAbsoluteTestRootDir(fc)+"/test1";
return fileContextTestHelper.getAbsoluteTestRootDir(fc)+"/test1";
}
@Override
protected String testBaseDir2() throws IOException {
return getAbsoluteTestRootDir(fc)+"/test2";
return fileContextTestHelper.getAbsoluteTestRootDir(fc)+"/test2";
}
@Override

6
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
View File

@ -72,7 +72,7 @@ public class TestLocalFileSystem {
/**
* Test the capability of setting the working directory.
*/
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testWorkingDirectory() throws IOException {
Path origDir = fileSys.getWorkingDirectory();
Path subdir = new Path(TEST_ROOT_DIR, "new");
@ -159,7 +159,7 @@ public class TestLocalFileSystem {
}
}
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testCopy() throws IOException {
Path src = new Path(TEST_ROOT_DIR, "dingo");
Path dst = new Path(TEST_ROOT_DIR, "yak");
@ -279,7 +279,7 @@ public class TestLocalFileSystem {
stats[0].getPath().toUri().getPath());
}
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testReportChecksumFailure() throws IOException {
base.mkdirs();
assertTrue(base.exists() && base.isDirectory());

6
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java
View File

@ -25,8 +25,6 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import static org.apache.hadoop.fs.FileContextTestHelper.*;
/**
* <p>
* Tests the File Context Statistics for {@link LocalFileSystem}
@ -39,12 +37,12 @@ public class TestLocalFsFCStatistics extends FCStatisticsBaseTest {
@Before
public void setUp() throws Exception {
fc = FileContext.getLocalFSFileContext();
fc.mkdir(getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true);
fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true);
}
@After
public void tearDown() throws Exception {
fc.delete(getTestRootPath(fc, "test"), true);
fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
}
@Override

12
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java
View File

@ -97,37 +97,37 @@ public class TestCopyPreserveFlag {
assertEquals(0, cmd.run(args));
}
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testPutWithP() throws Exception {
run(new Put(), "-p", FROM.toString(), TO.toString());
assertAttributesPreserved();
}
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testPutWithoutP() throws Exception {
run(new Put(), FROM.toString(), TO.toString());
assertAttributesChanged();
}
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testGetWithP() throws Exception {
run(new Get(), "-p", FROM.toString(), TO.toString());
assertAttributesPreserved();
}
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testGetWithoutP() throws Exception {
run(new Get(), FROM.toString(), TO.toString());
assertAttributesChanged();
}
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testCpWithP() throws Exception {
run(new Cp(), "-p", FROM.toString(), TO.toString());
assertAttributesPreserved();
}
@Test(timeout = 1000)
@Test(timeout = 10000)
public void testCpWithoutP() throws Exception {
run(new Cp(), FROM.toString(), TO.toString());
assertAttributesChanged();

30
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
View File

@ -40,13 +40,15 @@ public class TestChRootedFileSystem {
FileSystem fSys; // The ChRoootedFs
FileSystem fSysTarget; //
Path chrootedTo;
FileSystemTestHelper fileSystemTestHelper;
@Before
public void setUp() throws Exception {
// create the test root on local_fs
Configuration conf = new Configuration();
fSysTarget = FileSystem.getLocal(conf);
chrootedTo = FileSystemTestHelper.getAbsoluteTestRootPath(fSysTarget);
fileSystemTestHelper = new FileSystemTestHelper();
chrootedTo = fileSystemTestHelper.getAbsoluteTestRootPath(fSysTarget);
// In case previous test was killed before cleanup
fSysTarget.delete(chrootedTo, true);
@ -107,12 +109,12 @@ public class TestChRootedFileSystem {
// Create file
FileSystemTestHelper.createFile(fSys, "/foo");
fileSystemTestHelper.createFile(fSys, "/foo");
Assert.assertTrue(fSys.isFile(new Path("/foo")));
Assert.assertTrue(fSysTarget.isFile(new Path(chrootedTo, "foo")));
// Create file with recursive dir
FileSystemTestHelper.createFile(fSys, "/newDir/foo");
fileSystemTestHelper.createFile(fSys, "/newDir/foo");
Assert.assertTrue(fSys.isFile(new Path("/newDir/foo")));
Assert.assertTrue(fSysTarget.isFile(new Path(chrootedTo,"newDir/foo")));
@ -122,7 +124,7 @@ public class TestChRootedFileSystem {
Assert.assertFalse(fSysTarget.exists(new Path(chrootedTo, "newDir/foo")));
// Create file with a 2 component dirs recursively
FileSystemTestHelper.createFile(fSys, "/newDir/newDir2/foo");
fileSystemTestHelper.createFile(fSys, "/newDir/newDir2/foo");
Assert.assertTrue(fSys.isFile(new Path("/newDir/newDir2/foo")));
Assert.assertTrue(fSysTarget.isFile(new Path(chrootedTo,"newDir/newDir2/foo")));
@ -135,11 +137,11 @@ public class TestChRootedFileSystem {
@Test
public void testMkdirDelete() throws IOException {
fSys.mkdirs(FileSystemTestHelper.getTestRootPath(fSys, "/dirX"));
fSys.mkdirs(fileSystemTestHelper.getTestRootPath(fSys, "/dirX"));
Assert.assertTrue(fSys.isDirectory(new Path("/dirX")));
Assert.assertTrue(fSysTarget.isDirectory(new Path(chrootedTo,"dirX")));
fSys.mkdirs(FileSystemTestHelper.getTestRootPath(fSys, "/dirX/dirY"));
fSys.mkdirs(fileSystemTestHelper.getTestRootPath(fSys, "/dirX/dirY"));
Assert.assertTrue(fSys.isDirectory(new Path("/dirX/dirY")));
Assert.assertTrue(fSysTarget.isDirectory(new Path(chrootedTo,"dirX/dirY")));
@ -157,11 +159,11 @@ public class TestChRootedFileSystem {
@Test
public void testRename() throws IOException {
// Rename a file
FileSystemTestHelper.createFile(fSys, "/newDir/foo");
fileSystemTestHelper.createFile(fSys, "/newDir/foo");
fSys.rename(new Path("/newDir/foo"), new Path("/newDir/fooBar"));
Assert.assertFalse(fSys.exists(new Path("/newDir/foo")));
Assert.assertFalse(fSysTarget.exists(new Path(chrootedTo,"newDir/foo")));
Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/newDir/fooBar")));
Assert.assertTrue(fSys.isFile(fileSystemTestHelper.getTestRootPath(fSys,"/newDir/fooBar")));
Assert.assertTrue(fSysTarget.isFile(new Path(chrootedTo,"newDir/fooBar")));
@ -170,7 +172,7 @@ public class TestChRootedFileSystem {
fSys.rename(new Path("/newDir/dirFoo"), new Path("/newDir/dirFooBar"));
Assert.assertFalse(fSys.exists(new Path("/newDir/dirFoo")));
Assert.assertFalse(fSysTarget.exists(new Path(chrootedTo,"newDir/dirFoo")));
Assert.assertTrue(fSys.isDirectory(FileSystemTestHelper.getTestRootPath(fSys,"/newDir/dirFooBar")));
Assert.assertTrue(fSys.isDirectory(fileSystemTestHelper.getTestRootPath(fSys,"/newDir/dirFooBar")));
Assert.assertTrue(fSysTarget.isDirectory(new Path(chrootedTo,"newDir/dirFooBar")));
}
@ -214,10 +216,10 @@ public class TestChRootedFileSystem {
FileSystemTestHelper.createFile(fSys, "/foo");
FileSystemTestHelper.createFile(fSys, "/bar");
fileSystemTestHelper.createFile(fSys, "/foo");
fileSystemTestHelper.createFile(fSys, "/bar");
fSys.mkdirs(new Path("/dirX"));
fSys.mkdirs(FileSystemTestHelper.getTestRootPath(fSys, "/dirY"));
fSys.mkdirs(fileSystemTestHelper.getTestRootPath(fSys, "/dirY"));
fSys.mkdirs(new Path("/dirX/dirXX"));
dirPaths = fSys.listStatus(new Path("/"));
@ -307,7 +309,7 @@ public class TestChRootedFileSystem {
@Test
public void testResolvePath() throws IOException {
Assert.assertEquals(chrootedTo, fSys.resolvePath(new Path("/")));
FileSystemTestHelper.createFile(fSys, "/foo");
fileSystemTestHelper.createFile(fSys, "/foo");
Assert.assertEquals(new Path(chrootedTo, "foo"),
fSys.resolvePath(new Path("/foo")));
}

37
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
View File

@ -40,6 +40,7 @@ import org.junit.Test;
import org.mockito.Mockito;
public class TestChRootedFs {
FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
FileContext fc; // The ChRoootedFs
FileContext fcTarget; //
Path chrootedTo;
@ -48,7 +49,7 @@ public class TestChRootedFs {
public void setUp() throws Exception {
// create the test root on local_fs
fcTarget = FileContext.getLocalFSFileContext();
chrootedTo = FileContextTestHelper.getAbsoluteTestRootPath(fcTarget);
chrootedTo = fileContextTestHelper.getAbsoluteTestRootPath(fcTarget);
// In case previous test was killed before cleanup
fcTarget.delete(chrootedTo, true);
@ -107,12 +108,12 @@ public class TestChRootedFs {
// Create file
FileContextTestHelper.createFileNonRecursive(fc, "/foo");
fileContextTestHelper.createFileNonRecursive(fc, "/foo");
Assert.assertTrue(isFile(fc, new Path("/foo")));
Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo, "foo")));
// Create file with recursive dir
FileContextTestHelper.createFile(fc, "/newDir/foo");
fileContextTestHelper.createFile(fc, "/newDir/foo");
Assert.assertTrue(isFile(fc, new Path("/newDir/foo")));
Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo,"newDir/foo")));
@ -122,7 +123,7 @@ public class TestChRootedFs {
Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"newDir/foo")));
// Create file with a 2 component dirs recursively
FileContextTestHelper.createFile(fc, "/newDir/newDir2/foo");
fileContextTestHelper.createFile(fc, "/newDir/newDir2/foo");
Assert.assertTrue(isFile(fc, new Path("/newDir/newDir2/foo")));
Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo,"newDir/newDir2/foo")));
@ -135,11 +136,11 @@ public class TestChRootedFs {
@Test
public void testMkdirDelete() throws IOException {
fc.mkdir(FileContextTestHelper.getTestRootPath(fc, "/dirX"), FileContext.DEFAULT_PERM, false);
fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "/dirX"), FileContext.DEFAULT_PERM, false);
Assert.assertTrue(isDir(fc, new Path("/dirX")));
Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"dirX")));
fc.mkdir(FileContextTestHelper.getTestRootPath(fc, "/dirX/dirY"), FileContext.DEFAULT_PERM, false);
fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "/dirX/dirY"), FileContext.DEFAULT_PERM, false);
Assert.assertTrue(isDir(fc, new Path("/dirX/dirY")));
Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"dirX/dirY")));
@ -157,11 +158,11 @@ public class TestChRootedFs {
@Test
public void testRename() throws IOException {
// Rename a file
FileContextTestHelper.createFile(fc, "/newDir/foo");
fileContextTestHelper.createFile(fc, "/newDir/foo");
fc.rename(new Path("/newDir/foo"), new Path("/newDir/fooBar"));
Assert.assertFalse(exists(fc, new Path("/newDir/foo")));
Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"newDir/foo")));
Assert.assertTrue(isFile(fc, FileContextTestHelper.getTestRootPath(fc,"/newDir/fooBar")));
Assert.assertTrue(isFile(fc, fileContextTestHelper.getTestRootPath(fc,"/newDir/fooBar")));
Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo,"newDir/fooBar")));
@ -170,7 +171,7 @@ public class TestChRootedFs {
fc.rename(new Path("/newDir/dirFoo"), new Path("/newDir/dirFooBar"));
Assert.assertFalse(exists(fc, new Path("/newDir/dirFoo")));
Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"newDir/dirFoo")));
Assert.assertTrue(isDir(fc, FileContextTestHelper.getTestRootPath(fc,"/newDir/dirFooBar")));
Assert.assertTrue(isDir(fc, fileContextTestHelper.getTestRootPath(fc,"/newDir/dirFooBar")));
Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"newDir/dirFooBar")));
}
@ -204,10 +205,10 @@ public class TestChRootedFs {
FileContextTestHelper.createFileNonRecursive(fc, "/foo");
FileContextTestHelper.createFileNonRecursive(fc, "/bar");
fileContextTestHelper.createFileNonRecursive(fc, "/foo");
fileContextTestHelper.createFileNonRecursive(fc, "/bar");
fc.mkdir(new Path("/dirX"), FileContext.DEFAULT_PERM, false);
fc.mkdir(FileContextTestHelper.getTestRootPath(fc, "/dirY"),
fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "/dirY"),
FileContext.DEFAULT_PERM, false);
fc.mkdir(new Path("/dirX/dirXX"), FileContext.DEFAULT_PERM, false);
@ -215,16 +216,16 @@ public class TestChRootedFs {
Assert.assertEquals(4, dirPaths.length);
// Note the the file status paths are the full paths on target
fs = FileContextTestHelper.containsPath(fcTarget, "foo", dirPaths);
fs = fileContextTestHelper.containsPath(fcTarget, "foo", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue(fs.isFile());
fs = FileContextTestHelper.containsPath(fcTarget, "bar", dirPaths);
fs = fileContextTestHelper.containsPath(fcTarget, "bar", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue(fs.isFile());
fs = FileContextTestHelper.containsPath(fcTarget, "dirX", dirPaths);
fs = fileContextTestHelper.containsPath(fcTarget, "dirX", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue(fs.isDirectory());
fs = FileContextTestHelper.containsPath(fcTarget, "dirY", dirPaths);
fs = fileContextTestHelper.containsPath(fcTarget, "dirY", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue(fs.isDirectory());
}
@ -275,7 +276,7 @@ public class TestChRootedFs {
fc.mkdir(new Path("newDir"), FileContext.DEFAULT_PERM, true);
Assert.assertTrue(isDir(fc, new Path(absoluteDir, "newDir")));
absoluteDir = getTestRootPath(fc, "nonexistingPath");
absoluteDir = fileContextTestHelper.getTestRootPath(fc, "nonexistingPath");
try {
fc.setWorkingDirectory(absoluteDir);
Assert.fail("cd to non existing dir should have failed");
@ -299,7 +300,7 @@ public class TestChRootedFs {
@Test
public void testResolvePath() throws IOException {
Assert.assertEquals(chrootedTo, fc.getDefaultFileSystem().resolvePath(new Path("/")));
FileContextTestHelper.createFile(fc, "/foo");
fileContextTestHelper.createFile(fc, "/foo");
Assert.assertEquals(new Path(chrootedTo, "foo"),
fc.getDefaultFileSystem().resolvePath(new Path("/foo")));
}

17
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java
View File

@ -23,7 +23,6 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSMainOperationsBaseTest;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
@ -32,14 +31,19 @@ import org.junit.Before;
import org.junit.Test;
public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTest {
static FileSystem fcTarget;
FileSystem fcTarget;
@Override
protected FileSystem createFileSystem() throws Exception {
return ViewFileSystemTestSetup.setupForViewFileSystem(
ViewFileSystemTestSetup.createConfig(), this, fcTarget);
}
@Override
@Before
public void setUp() throws Exception {
Configuration conf = new Configuration();
fcTarget = FileSystem.getLocal(conf);
fSys = ViewFileSystemTestSetup.setupForViewFileSystem(
ViewFileSystemTestSetup.createConfig(), fcTarget);
super.setUp();
}
@ -47,14 +51,13 @@ public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTes
@After
public void tearDown() throws Exception {
super.tearDown();
ViewFileSystemTestSetup.tearDown(fcTarget);
ViewFileSystemTestSetup.tearDown(this, fcTarget);
}
@Test
@Override
public void testWDAbsolute() throws IOException {
Path absoluteDir = FileSystemTestHelper.getTestRootPath(fSys,
"test/existingDir");
Path absoluteDir = getTestRootPath(fSys, "test/existingDir");
fSys.mkdirs(absoluteDir);
fSys.setWorkingDirectory(absoluteDir);
Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcCreateMkdirLocalFs.java
View File

@ -31,7 +31,7 @@ public class TestFcCreateMkdirLocalFs extends
@Override
@Before
public void setUp() throws Exception {
fc = ViewFsTestSetup.setupForViewFsLocalFs();
fc = ViewFsTestSetup.setupForViewFsLocalFs(fileContextTestHelper);
super.setUp();
}
@ -39,6 +39,6 @@ public class TestFcCreateMkdirLocalFs extends
@After
public void tearDown() throws Exception {
super.tearDown();
ViewFsTestSetup.tearDownForViewFsLocalFs();
ViewFsTestSetup.tearDownForViewFsLocalFs(fileContextTestHelper);
}
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java
View File

@ -39,7 +39,7 @@ public class TestFcMainOperationsLocalFs extends
@Override
@Before
public void setUp() throws Exception {
fc = ViewFsTestSetup.setupForViewFsLocalFs();
fc = ViewFsTestSetup.setupForViewFsLocalFs(fileContextTestHelper);
super.setUp();
}
@ -47,6 +47,6 @@ public class TestFcMainOperationsLocalFs extends
@After
public void tearDown() throws Exception {
super.tearDown();
ViewFsTestSetup.tearDownForViewFsLocalFs();
ViewFsTestSetup.tearDownForViewFsLocalFs(fileContextTestHelper);
}
}

10
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcPermissionsLocalFs.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.fs.viewfs;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextPermissionBase;
import org.junit.After;
@ -26,11 +27,9 @@ import org.junit.Before;
public class TestFcPermissionsLocalFs extends FileContextPermissionBase {
@Override
@Before
public void setUp() throws Exception {
fc = ViewFsTestSetup.setupForViewFsLocalFs();
super.setUp();
}
@ -38,6 +37,11 @@ public class TestFcPermissionsLocalFs extends FileContextPermissionBase {
@After
public void tearDown() throws Exception {
super.tearDown();
ViewFsTestSetup.tearDownForViewFsLocalFs();
ViewFsTestSetup.tearDownForViewFsLocalFs(fileContextTestHelper);
}
@Override
protected FileContext getFileContext() throws Exception {
return ViewFsTestSetup.setupForViewFsLocalFs(fileContextTestHelper);
}
}

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java
View File

@ -52,7 +52,7 @@ public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup {
conf.set("fs."+scheme+".impl", clazz.getName());
FakeFileSystem fs = (FakeFileSystem)FileSystem.get(uri, conf);
assertEquals(uri, fs.getUri());
Path targetPath = FileSystemTestHelper.getAbsoluteTestRootPath(fs);
Path targetPath = new FileSystemTestHelper().getAbsoluteTestRootPath(fs);
ConfigUtil.addLink(conf, "/mounts/"+scheme, targetPath.toUri());
return fs;
}

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java
View File

@ -38,7 +38,6 @@ import org.junit.Before;
public class TestViewFileSystemLocalFileSystem extends ViewFileSystemBaseTest {
@Override
@Before
public void setUp() throws Exception {
@ -51,7 +50,7 @@ public class TestViewFileSystemLocalFileSystem extends ViewFileSystemBaseTest {
@Override
@After
public void tearDown() throws Exception {
fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true);
fsTarget.delete(fileSystemTestHelper.getTestRootPath(fsTarget), true);
super.tearDown();
}
}

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java
View File

@ -59,7 +59,7 @@ public class TestViewFileSystemWithAuthorityLocalFileSystem extends ViewFileSyst
@Override
@After
public void tearDown() throws Exception {
fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true);
fsTarget.delete(fileSystemTestHelper.getTestRootPath(fsTarget), true);
super.tearDown();
}

13
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java
View File

@ -35,11 +35,12 @@ public class TestViewFsTrash {
FileSystem fsTarget; // the target file system - the mount will point here
FileSystem fsView;
Configuration conf;
FileSystemTestHelper fileSystemTestHelper = new FileSystemTestHelper();
static class TestLFS extends LocalFileSystem {
class TestLFS extends LocalFileSystem {
Path home;
TestLFS() throws IOException {
this(new Path(FileSystemTestHelper.TEST_ROOT_DIR));
this(new Path(fileSystemTestHelper.getTestRootDir()));
}
TestLFS(Path home) throws IOException {
super();
@ -54,23 +55,23 @@ public class TestViewFsTrash {
@Before
public void setUp() throws Exception {
fsTarget = FileSystem.getLocal(new Configuration());
fsTarget.mkdirs(new Path(FileSystemTestHelper.
fsTarget.mkdirs(new Path(fileSystemTestHelper.
getTestRootPath(fsTarget), "dir1"));
conf = ViewFileSystemTestSetup.createConfig();
fsView = ViewFileSystemTestSetup.setupForViewFileSystem(conf, fsTarget);
fsView = ViewFileSystemTestSetup.setupForViewFileSystem(conf, fileSystemTestHelper, fsTarget);
conf.set("fs.defaultFS", FsConstants.VIEWFS_URI.toString());
}
@After
public void tearDown() throws Exception {
ViewFileSystemTestSetup.tearDown(fsTarget);
ViewFileSystemTestSetup.tearDown(fileSystemTestHelper, fsTarget);
fsTarget.delete(new Path(fsTarget.getHomeDirectory(), ".Trash/Current"),
true);
}
@Test
public void testTrash() throws IOException {
TestTrash.trashShell(conf, FileSystemTestHelper.getTestRootPath(fsView),
TestTrash.trashShell(conf, fileSystemTestHelper.getTestRootPath(fsView),
fsTarget, new Path(fsTarget.getHomeDirectory(), ".Trash/Current"));
}

106
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -68,7 +68,15 @@ public class ViewFileSystemBaseTest {
FileSystem fsTarget; // the target file system - the mount will point here
Path targetTestRoot;
Configuration conf;
final FileSystemTestHelper fileSystemTestHelper;
public ViewFileSystemBaseTest() {
this.fileSystemTestHelper = createFileSystemHelper();
}
protected FileSystemTestHelper createFileSystemHelper() {
return new FileSystemTestHelper();
}
@Before
public void setUp() throws Exception {
@ -94,11 +102,11 @@ public class ViewFileSystemBaseTest {
@After
public void tearDown() throws Exception {
fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true);
fsTarget.delete(fileSystemTestHelper.getTestRootPath(fsTarget), true);
}
void initializeTargetTestRoot() throws IOException {
targetTestRoot = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.delete(targetTestRoot, true);
@ -199,7 +207,7 @@ public class ViewFileSystemBaseTest {
@Test
public void testOperationsThroughMountLinks() throws IOException {
// Create file
FileSystemTestHelper.createFile(fsView, "/user/foo");
fileSystemTestHelper.createFile(fsView, "/user/foo");
Assert.assertTrue("Created file should be type file",
fsView.isFile(new Path("/user/foo")));
Assert.assertTrue("Target of created file should be type file",
@ -214,7 +222,7 @@ public class ViewFileSystemBaseTest {
fsTarget.exists(new Path(targetTestRoot,"user/foo")));
// Create file with a 2 component dirs
FileSystemTestHelper.createFile(fsView, "/internalDir/linkToDir2/foo");
fileSystemTestHelper.createFile(fsView, "/internalDir/linkToDir2/foo");
Assert.assertTrue("Created file should be type file",
fsView.isFile(new Path("/internalDir/linkToDir2/foo")));
Assert.assertTrue("Target of created file should be type file",
@ -230,14 +238,14 @@ public class ViewFileSystemBaseTest {
// Create file with a 3 component dirs
FileSystemTestHelper.createFile(fsView, "/internalDir/internalDir2/linkToDir3/foo");
fileSystemTestHelper.createFile(fsView, "/internalDir/internalDir2/linkToDir3/foo");
Assert.assertTrue("Created file should be type file",
fsView.isFile(new Path("/internalDir/internalDir2/linkToDir3/foo")));
Assert.assertTrue("Target of created file should be type file",
fsTarget.isFile(new Path(targetTestRoot,"dir3/foo")));
// Recursive Create file with missing dirs
FileSystemTestHelper.createFile(fsView,
fileSystemTestHelper.createFile(fsView,
"/internalDir/linkToDir2/missingDir/miss2/foo");
Assert.assertTrue("Created file should be type file",
fsView.isFile(new Path("/internalDir/linkToDir2/missingDir/miss2/foo")));
@ -256,14 +264,14 @@ public class ViewFileSystemBaseTest {
// mkdir
fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX"));
fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX"));
Assert.assertTrue("New dir should be type dir",
fsView.isDirectory(new Path("/user/dirX")));
Assert.assertTrue("Target of new dir should be of type dir",
fsTarget.isDirectory(new Path(targetTestRoot,"user/dirX")));
fsView.mkdirs(
FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX/dirY"));
fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX/dirY"));
Assert.assertTrue("New dir should be type dir",
fsView.isDirectory(new Path("/user/dirX/dirY")));
Assert.assertTrue("Target of new dir should be of type dir",
@ -285,14 +293,14 @@ public class ViewFileSystemBaseTest {
Assert.assertFalse(fsTarget.exists(new Path(targetTestRoot,"user/dirX")));
// Rename a file
FileSystemTestHelper.createFile(fsView, "/user/foo");
fileSystemTestHelper.createFile(fsView, "/user/foo");
fsView.rename(new Path("/user/foo"), new Path("/user/fooBar"));
Assert.assertFalse("Renamed src should not exist",
fsView.exists(new Path("/user/foo")));
Assert.assertFalse("Renamed src should not exist in target",
fsTarget.exists(new Path(targetTestRoot,"user/foo")));
Assert.assertTrue("Renamed dest should exist as file",
fsView.isFile(FileSystemTestHelper.getTestRootPath(fsView,"/user/fooBar")));
fsView.isFile(fileSystemTestHelper.getTestRootPath(fsView,"/user/fooBar")));
Assert.assertTrue("Renamed dest should exist as file in target",
fsTarget.isFile(new Path(targetTestRoot,"user/fooBar")));
@ -303,7 +311,7 @@ public class ViewFileSystemBaseTest {
Assert.assertFalse("Renamed src should not exist in target",
fsTarget.exists(new Path(targetTestRoot,"user/dirFoo")));
Assert.assertTrue("Renamed dest should exist as dir",
fsView.isDirectory(FileSystemTestHelper.getTestRootPath(fsView,"/user/dirFooBar")));
fsView.isDirectory(fileSystemTestHelper.getTestRootPath(fsView,"/user/dirFooBar")));
Assert.assertTrue("Renamed dest should exist as dir in target",
fsTarget.isDirectory(new Path(targetTestRoot,"user/dirFooBar")));
@ -322,7 +330,7 @@ public class ViewFileSystemBaseTest {
// rename across mount points that point to same target also fail
@Test(expected=IOException.class)
public void testRenameAcrossMounts1() throws IOException {
FileSystemTestHelper.createFile(fsView, "/user/foo");
fileSystemTestHelper.createFile(fsView, "/user/foo");
fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
/* - code if we had wanted this to suceed
Assert.assertFalse(fSys.exists(new Path("/user/foo")));
@ -338,7 +346,7 @@ public class ViewFileSystemBaseTest {
@Test(expected=IOException.class)
public void testRenameAcrossMounts2() throws IOException {
FileSystemTestHelper.createFile(fsView, "/user/foo");
fileSystemTestHelper.createFile(fsView, "/user/foo");
fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
}
@ -394,19 +402,19 @@ public class ViewFileSystemBaseTest {
FileStatus[] dirPaths = fsView.listStatus(new Path("/"));
FileStatus fs;
Assert.assertEquals(getExpectedDirPaths(), dirPaths.length);
fs = FileSystemTestHelper.containsPath(fsView, "/user", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/user", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
fs = FileSystemTestHelper.containsPath(fsView, "/data", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/data", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
fs = FileSystemTestHelper.containsPath(fsView, "/internalDir", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/internalDir", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isDirectory());
fs = FileSystemTestHelper.containsPath(fsView, "/danglingLink", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/danglingLink", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
fs = FileSystemTestHelper.containsPath(fsView, "/linkToAFile", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/linkToAFile", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
@ -416,10 +424,10 @@ public class ViewFileSystemBaseTest {
dirPaths = fsView.listStatus(new Path("/internalDir"));
Assert.assertEquals(2, dirPaths.length);
fs = FileSystemTestHelper.containsPath(fsView, "/internalDir/internalDir2", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/internalDir/internalDir2", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isDirectory());
fs = FileSystemTestHelper.containsPath(fsView, "/internalDir/linkToDir2",
fs = fileSystemTestHelper.containsPath(fsView, "/internalDir/linkToDir2",
dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
@ -436,22 +444,22 @@ public class ViewFileSystemBaseTest {
Assert.assertEquals(0, dirPaths.length);
// add a file
long len = FileSystemTestHelper.createFile(fsView, "/data/foo");
long len = fileSystemTestHelper.createFile(fsView, "/data/foo");
dirPaths = fsView.listStatus(new Path("/data"));
Assert.assertEquals(1, dirPaths.length);
fs = FileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("Created file shoudl appear as a file", fs.isFile());
Assert.assertEquals(len, fs.getLen());
// add a dir
fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/data/dirX"));
fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/data/dirX"));
dirPaths = fsView.listStatus(new Path("/data"));
Assert.assertEquals(2, dirPaths.length);
fs = FileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("Created file shoudl appear as a file", fs.isFile());
fs = FileSystemTestHelper.containsPath(fsView, "/data/dirX", dirPaths);
fs = fileSystemTestHelper.containsPath(fsView, "/data/dirX", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("Created dir should appear as a dir", fs.isDirectory());
}
@ -487,12 +495,12 @@ public class ViewFileSystemBaseTest {
public void testResolvePathInternalPaths() throws IOException {
Assert.assertEquals(new Path("/"), fsView.resolvePath(new Path("/")));
Assert.assertEquals(new Path("/internalDir"),
fsView.resolvePath(new Path("/internalDir")));
fsView.resolvePath(new Path("/internalDir")));
}
@Test
public void testResolvePathMountPoints() throws IOException {
Assert.assertEquals(new Path(targetTestRoot,"user"),
fsView.resolvePath(new Path("/user")));
fsView.resolvePath(new Path("/user")));
Assert.assertEquals(new Path(targetTestRoot,"data"),
fsView.resolvePath(new Path("/data")));
Assert.assertEquals(new Path(targetTestRoot,"dir2"),
@ -504,25 +512,25 @@ public class ViewFileSystemBaseTest {
@Test
public void testResolvePathThroughMountPoints() throws IOException {
FileSystemTestHelper.createFile(fsView, "/user/foo");
fileSystemTestHelper.createFile(fsView, "/user/foo");
Assert.assertEquals(new Path(targetTestRoot,"user/foo"),
fsView.resolvePath(new Path("/user/foo")));
fsView.resolvePath(new Path("/user/foo")));
fsView.mkdirs(
FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX"));
fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX"));
Assert.assertEquals(new Path(targetTestRoot,"user/dirX"),
fsView.resolvePath(new Path("/user/dirX")));
fsView.mkdirs(
FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX/dirY"));
fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX/dirY"));
Assert.assertEquals(new Path(targetTestRoot,"user/dirX/dirY"),
fsView.resolvePath(new Path("/user/dirX/dirY")));
}
@Test(expected=FileNotFoundException.class)
public void testResolvePathDanglingLink() throws IOException {
fsView.resolvePath(new Path("/danglingLink"));
fsView.resolvePath(new Path("/danglingLink"));
}
@Test(expected=FileNotFoundException.class)
@ -534,7 +542,7 @@ public class ViewFileSystemBaseTest {
@Test(expected=FileNotFoundException.class)
public void testResolvePathMissingThroughMountPoints2() throws IOException {
fsView.mkdirs(
FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX"));
fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX"));
fsView.resolvePath(new Path("/user/dirX/nonExisting"));
}
@ -550,76 +558,76 @@ public class ViewFileSystemBaseTest {
// Mkdir on existing internal mount table succeed except for /
@Test(expected=AccessControlException.class)
public void testInternalMkdirSlash() throws IOException {
fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/"));
fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/"));
}
public void testInternalMkdirExisting1() throws IOException {
Assert.assertTrue("mkdir of existing dir should succeed",
fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView,
fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView,
"/internalDir")));
}
public void testInternalMkdirExisting2() throws IOException {
Assert.assertTrue("mkdir of existing dir should succeed",
fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView,
fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView,
"/internalDir/linkToDir2")));
}
// Mkdir for new internal mount table should fail
@Test(expected=AccessControlException.class)
public void testInternalMkdirNew() throws IOException {
fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/dirNew"));
fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/dirNew"));
}
@Test(expected=AccessControlException.class)
public void testInternalMkdirNew2() throws IOException {
fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/internalDir/dirNew"));
fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/internalDir/dirNew"));
}
// Create File on internal mount table should fail
@Test(expected=AccessControlException.class)
public void testInternalCreate1() throws IOException {
FileSystemTestHelper.createFile(fsView, "/foo"); // 1 component
fileSystemTestHelper.createFile(fsView, "/foo"); // 1 component
}
@Test(expected=AccessControlException.class)
public void testInternalCreate2() throws IOException { // 2 component
FileSystemTestHelper.createFile(fsView, "/internalDir/foo");
fileSystemTestHelper.createFile(fsView, "/internalDir/foo");
}
@Test(expected=AccessControlException.class)
public void testInternalCreateMissingDir() throws IOException {
FileSystemTestHelper.createFile(fsView, "/missingDir/foo");
fileSystemTestHelper.createFile(fsView, "/missingDir/foo");
}
@Test(expected=AccessControlException.class)
public void testInternalCreateMissingDir2() throws IOException {
FileSystemTestHelper.createFile(fsView, "/missingDir/miss2/foo");
fileSystemTestHelper.createFile(fsView, "/missingDir/miss2/foo");
}
@Test(expected=AccessControlException.class)
public void testInternalCreateMissingDir3() throws IOException {
FileSystemTestHelper.createFile(fsView, "/internalDir/miss2/foo");
fileSystemTestHelper.createFile(fsView, "/internalDir/miss2/foo");
}
// Delete on internal mount table should fail
@Test(expected=FileNotFoundException.class)
public void testInternalDeleteNonExisting() throws IOException {
fsView.delete(new Path("/NonExisting"), false);
fsView.delete(new Path("/NonExisting"), false);
}
@Test(expected=FileNotFoundException.class)
public void testInternalDeleteNonExisting2() throws IOException {
fsView.delete(new Path("/internalDir/NonExisting"), false);
fsView.delete(new Path("/internalDir/NonExisting"), false);
}
@Test(expected=AccessControlException.class)
public void testInternalDeleteExisting() throws IOException {
fsView.delete(new Path("/internalDir"), false);
fsView.delete(new Path("/internalDir"), false);
}
@Test(expected=AccessControlException.class)
public void testInternalDeleteExisting2() throws IOException {
fsView.getFileStatus(
fsView.getFileStatus(
new Path("/internalDir/linkToDir2")).isDirectory();
fsView.delete(new Path("/internalDir/linkToDir2"), false);
}
@ -641,7 +649,7 @@ public class ViewFileSystemBaseTest {
}
@Test(expected=AccessControlException.class)
public void testInternalRename2() throws IOException {
fsView.getFileStatus(new Path("/internalDir/linkToDir2")).isDirectory();
fsView.getFileStatus(new Path("/internalDir/linkToDir2")).isDirectory();
fsView.rename(new Path("/internalDir/linkToDir2"),
new Path("/internalDir/dir1"));
}
@ -665,7 +673,7 @@ public class ViewFileSystemBaseTest {
@Test
public void testCreateNonRecursive() throws IOException {
Path path = FileSystemTestHelper.getTestRootPath(fsView, "/user/foo");
Path path = fileSystemTestHelper.getTestRootPath(fsView, "/user/foo");
fsView.createNonRecursive(path, false, 1024, (short)1, 1024L, null);
FileStatus status = fsView.getFileStatus(new Path("/user/foo"));
Assert.assertTrue("Created file should be type file",

17
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java
View File

@ -56,18 +56,18 @@ public class ViewFileSystemTestSetup {
* @return return the ViewFS File context to be used for tests
* @throws Exception
*/
static public FileSystem setupForViewFileSystem(Configuration conf, FileSystem fsTarget) throws Exception {
static public FileSystem setupForViewFileSystem(Configuration conf, FileSystemTestHelper fileSystemTestHelper, FileSystem fsTarget) throws Exception {
/**
* create the test root on local_fs - the mount table will point here
*/
Path targetOfTests = FileSystemTestHelper.getTestRootPath(fsTarget);
Path targetOfTests = fileSystemTestHelper.getTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.delete(targetOfTests, true);
fsTarget.mkdirs(targetOfTests);
// Set up viewfs link for test dir as described above
String testDir = FileSystemTestHelper.getTestRootPath(fsTarget).toUri()
String testDir = fileSystemTestHelper.getTestRootPath(fsTarget).toUri()
.getPath();
linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
@ -92,14 +92,21 @@ public class ViewFileSystemTestSetup {
*
* delete the test directory in the target fs
*/
static public void tearDown(FileSystem fsTarget) throws Exception {
Path targetOfTests = FileSystemTestHelper.getTestRootPath(fsTarget);
static public void tearDown(FileSystemTestHelper fileSystemTestHelper, FileSystem fsTarget) throws Exception {
Path targetOfTests = fileSystemTestHelper.getTestRootPath(fsTarget);
fsTarget.delete(targetOfTests, true);
}
public static Configuration createConfig() {
return createConfig(true);
}
public static Configuration createConfig(boolean disableCache) {
Configuration conf = new Configuration();
conf.set("fs.viewfs.impl", ViewFileSystem.class.getName());
if (disableCache) {
conf.set("fs.viewfs.impl.disable.cache", "true");
}
return conf;
}

69
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
View File

@ -75,6 +75,7 @@ public class ViewFsBaseTest {
Configuration conf;
FileContext xfcViewWithAuthority; // same as fsView but with authority
URI schemeWithAuthority;
final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
@Before
public void setUp() throws Exception {
@ -118,7 +119,7 @@ public class ViewFsBaseTest {
}
void initializeTargetTestRoot() throws IOException {
targetTestRoot = FileContextTestHelper.getAbsoluteTestRootPath(fcTarget);
targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fcTarget);
// In case previous test was killed before cleanup
fcTarget.delete(targetTestRoot, true);
@ -127,7 +128,7 @@ public class ViewFsBaseTest {
@After
public void tearDown() throws Exception {
fcTarget.delete(FileContextTestHelper.getTestRootPath(fcTarget), true);
fcTarget.delete(fileContextTestHelper.getTestRootPath(fcTarget), true);
}
@Test
@ -181,7 +182,7 @@ public class ViewFsBaseTest {
@Test
public void testOperationsThroughMountLinks() throws IOException {
// Create file
FileContextTestHelper.createFileNonRecursive(fcView, "/user/foo");
fileContextTestHelper.createFileNonRecursive(fcView, "/user/foo");
Assert.assertTrue("Create file should be file",
isFile(fcView, new Path("/user/foo")));
Assert.assertTrue("Target of created file should be type file",
@ -196,7 +197,7 @@ public class ViewFsBaseTest {
exists(fcTarget, new Path(targetTestRoot,"user/foo")));
// Create file with a 2 component dirs
FileContextTestHelper.createFileNonRecursive(fcView,
fileContextTestHelper.createFileNonRecursive(fcView,
"/internalDir/linkToDir2/foo");
Assert.assertTrue("Created file should be type file",
isFile(fcView, new Path("/internalDir/linkToDir2/foo")));
@ -213,7 +214,7 @@ public class ViewFsBaseTest {
// Create file with a 3 component dirs
FileContextTestHelper.createFileNonRecursive(fcView,
fileContextTestHelper.createFileNonRecursive(fcView,
"/internalDir/internalDir2/linkToDir3/foo");
Assert.assertTrue("Created file should be of type file",
isFile(fcView, new Path("/internalDir/internalDir2/linkToDir3/foo")));
@ -221,7 +222,7 @@ public class ViewFsBaseTest {
isFile(fcTarget, new Path(targetTestRoot,"dir3/foo")));
// Recursive Create file with missing dirs
FileContextTestHelper.createFile(fcView,
fileContextTestHelper.createFile(fcView,
"/internalDir/linkToDir2/missingDir/miss2/foo");
Assert.assertTrue("Created file should be of type file",
isFile(fcView, new Path("/internalDir/linkToDir2/missingDir/miss2/foo")));
@ -239,14 +240,14 @@ public class ViewFsBaseTest {
// mkdir
fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/user/dirX"),
fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/user/dirX"),
FileContext.DEFAULT_PERM, false);
Assert.assertTrue("New dir should be type dir",
isDir(fcView, new Path("/user/dirX")));
Assert.assertTrue("Target of new dir should be of type dir",
isDir(fcTarget, new Path(targetTestRoot,"user/dirX")));
fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"),
fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"),
FileContext.DEFAULT_PERM, false);
Assert.assertTrue("New dir should be type dir",
isDir(fcView, new Path("/user/dirX/dirY")));
@ -270,13 +271,13 @@ public class ViewFsBaseTest {
exists(fcTarget, new Path(targetTestRoot,"user/dirX")));
// Rename a file
FileContextTestHelper.createFile(fcView, "/user/foo");
fileContextTestHelper.createFile(fcView, "/user/foo");
fcView.rename(new Path("/user/foo"), new Path("/user/fooBar"));
Assert.assertFalse("Renamed src should not exist",
exists(fcView, new Path("/user/foo")));
Assert.assertFalse(exists(fcTarget, new Path(targetTestRoot,"user/foo")));
Assert.assertTrue(isFile(fcView,
FileContextTestHelper.getTestRootPath(fcView,"/user/fooBar")));
fileContextTestHelper.getTestRootPath(fcView,"/user/fooBar")));
Assert.assertTrue(isFile(fcTarget, new Path(targetTestRoot,"user/fooBar")));
fcView.mkdir(new Path("/user/dirFoo"), FileContext.DEFAULT_PERM, false);
@ -287,7 +288,7 @@ public class ViewFsBaseTest {
exists(fcTarget, new Path(targetTestRoot,"user/dirFoo")));
Assert.assertTrue("Renamed dest should exist as dir",
isDir(fcView,
FileContextTestHelper.getTestRootPath(fcView,"/user/dirFooBar")));
fileContextTestHelper.getTestRootPath(fcView,"/user/dirFooBar")));
Assert.assertTrue("Renamed dest should exist as dir in target",
isDir(fcTarget,new Path(targetTestRoot,"user/dirFooBar")));
@ -309,7 +310,7 @@ public class ViewFsBaseTest {
// rename across mount points that point to same target also fail
@Test(expected=IOException.class)
public void testRenameAcrossMounts1() throws IOException {
FileContextTestHelper.createFile(fcView, "/user/foo");
fileContextTestHelper.createFile(fcView, "/user/foo");
fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
/* - code if we had wanted this to succeed
Assert.assertFalse(exists(fc, new Path("/user/foo")));
@ -327,7 +328,7 @@ public class ViewFsBaseTest {
@Test(expected=IOException.class)
public void testRenameAcrossMounts2() throws IOException {
FileContextTestHelper.createFile(fcView, "/user/foo");
fileContextTestHelper.createFile(fcView, "/user/foo");
fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
}
@ -382,19 +383,19 @@ public class ViewFsBaseTest {
FileStatus[] dirPaths = fcView.util().listStatus(new Path("/"));
FileStatus fs;
Assert.assertEquals(7, dirPaths.length);
fs = FileContextTestHelper.containsPath(fcView, "/user", dirPaths);
fs = fileContextTestHelper.containsPath(fcView, "/user", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
fs = FileContextTestHelper.containsPath(fcView, "/data", dirPaths);
fs = fileContextTestHelper.containsPath(fcView, "/data", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
fs = FileContextTestHelper.containsPath(fcView, "/internalDir", dirPaths);
fs = fileContextTestHelper.containsPath(fcView, "/internalDir", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("InternalDirs should appear as dir", fs.isDirectory());
fs = FileContextTestHelper.containsPath(fcView, "/danglingLink", dirPaths);
fs = fileContextTestHelper.containsPath(fcView, "/danglingLink", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
fs = FileContextTestHelper.containsPath(fcView, "/linkToAFile", dirPaths);
fs = fileContextTestHelper.containsPath(fcView, "/linkToAFile", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
@ -404,11 +405,11 @@ public class ViewFsBaseTest {
dirPaths = fcView.util().listStatus(new Path("/internalDir"));
Assert.assertEquals(2, dirPaths.length);
fs = FileContextTestHelper.containsPath(fcView,
fs = fileContextTestHelper.containsPath(fcView,
"/internalDir/internalDir2", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("InternalDirs should appear as dir",fs.isDirectory());
fs = FileContextTestHelper.containsPath(fcView,
fs = fileContextTestHelper.containsPath(fcView,
"/internalDir/linkToDir2", dirPaths);
Assert.assertNotNull(fs);
Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
@ -534,19 +535,19 @@ public class ViewFsBaseTest {
@Test
public void testResolvePathThroughMountPoints() throws IOException {
FileContextTestHelper.createFile(fcView, "/user/foo");
fileContextTestHelper.createFile(fcView, "/user/foo");
Assert.assertEquals(new Path(targetTestRoot,"user/foo"),
fcView.resolvePath(new Path("/user/foo")));
fcView.mkdir(
FileContextTestHelper.getTestRootPath(fcView, "/user/dirX"),
fileContextTestHelper.getTestRootPath(fcView, "/user/dirX"),
FileContext.DEFAULT_PERM, false);
Assert.assertEquals(new Path(targetTestRoot,"user/dirX"),
fcView.resolvePath(new Path("/user/dirX")));
fcView.mkdir(
FileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"),
fileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"),
FileContext.DEFAULT_PERM, false);
Assert.assertEquals(new Path(targetTestRoot,"user/dirX/dirY"),
fcView.resolvePath(new Path("/user/dirX/dirY")));
@ -566,7 +567,7 @@ public class ViewFsBaseTest {
@Test(expected=FileNotFoundException.class)
public void testResolvePathMissingThroughMountPoints2() throws IOException {
fcView.mkdir(
FileContextTestHelper.getTestRootPath(fcView, "/user/dirX"),
fileContextTestHelper.getTestRootPath(fcView, "/user/dirX"),
FileContext.DEFAULT_PERM, false);
fcView.resolvePath(new Path("/user/dirX/nonExisting"));
}
@ -584,29 +585,29 @@ public class ViewFsBaseTest {
// Mkdir on internal mount table should fail
@Test(expected=AccessControlException.class)
public void testInternalMkdirSlash() throws IOException {
fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/"),
fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/"),
FileContext.DEFAULT_PERM, false);
}
@Test(expected=AccessControlException.class)
public void testInternalMkdirExisting1() throws IOException {
fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/internalDir"),
fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/internalDir"),
FileContext.DEFAULT_PERM, false);
}
@Test(expected=AccessControlException.class)
public void testInternalMkdirExisting2() throws IOException {
fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView,
fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView,
"/internalDir/linkToDir2"),
FileContext.DEFAULT_PERM, false);
}
@Test(expected=AccessControlException.class)
public void testInternalMkdirNew() throws IOException {
fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/dirNew"),
fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/dirNew"),
FileContext.DEFAULT_PERM, false);
}
@Test(expected=AccessControlException.class)
public void testInternalMkdirNew2() throws IOException {
fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/internalDir/dirNew"),
fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/internalDir/dirNew"),
FileContext.DEFAULT_PERM, false);
}
@ -614,28 +615,28 @@ public class ViewFsBaseTest {
@Test(expected=AccessControlException.class)
public void testInternalCreate1() throws IOException {
FileContextTestHelper.createFileNonRecursive(fcView, "/foo"); // 1 component
fileContextTestHelper.createFileNonRecursive(fcView, "/foo"); // 1 component
}
@Test(expected=AccessControlException.class)
public void testInternalCreate2() throws IOException { // 2 component
FileContextTestHelper.createFileNonRecursive(fcView, "/internalDir/foo");
fileContextTestHelper.createFileNonRecursive(fcView, "/internalDir/foo");
}
@Test(expected=AccessControlException.class)
public void testInternalCreateMissingDir() throws IOException {
FileContextTestHelper.createFile(fcView, "/missingDir/foo");
fileContextTestHelper.createFile(fcView, "/missingDir/foo");
}
@Test(expected=AccessControlException.class)
public void testInternalCreateMissingDir2() throws IOException {
FileContextTestHelper.createFile(fcView, "/missingDir/miss2/foo");
fileContextTestHelper.createFile(fcView, "/missingDir/miss2/foo");
}
@Test(expected=AccessControlException.class)
public void testInternalCreateMissingDir3() throws IOException {
FileContextTestHelper.createFile(fcView, "/internalDir/miss2/foo");
fileContextTestHelper.createFile(fcView, "/internalDir/miss2/foo");
}
// Delete on internal mount table should fail

10
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java
View File

@ -54,12 +54,12 @@ public class ViewFsTestSetup {
/*
* return the ViewFS File context to be used for tests
*/
static public FileContext setupForViewFsLocalFs() throws Exception {
static public FileContext setupForViewFsLocalFs(FileContextTestHelper helper) throws Exception {
/**
* create the test root on local_fs - the mount table will point here
*/
FileContext fsTarget = FileContext.getLocalFSFileContext();
Path targetOfTests = FileContextTestHelper.getTestRootPath(fsTarget);
Path targetOfTests = helper.getTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.delete(targetOfTests, true);
@ -67,7 +67,7 @@ public class ViewFsTestSetup {
Configuration conf = new Configuration();
// Set up viewfs link for test dir as described above
String testDir = FileContextTestHelper.getTestRootPath(fsTarget).toUri()
String testDir = helper.getTestRootPath(fsTarget).toUri()
.getPath();
linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
@ -92,9 +92,9 @@ public class ViewFsTestSetup {
*
* delete the test directory in the target local fs
*/
static public void tearDownForViewFsLocalFs() throws Exception {
static public void tearDownForViewFsLocalFs(FileContextTestHelper helper) throws Exception {
FileContext fclocal = FileContext.getLocalFSFileContext();
Path targetOfTests = FileContextTestHelper.getTestRootPath(fclocal);
Path targetOfTests = helper.getTestRootPath(fclocal);
fclocal.delete(targetOfTests, true);
}

46
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java
View File

@ -23,7 +23,10 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.Socket;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
@ -63,17 +66,13 @@ public abstract class ClientBaseWithFixes extends ZKTestCase {
static final File BASETEST =
new File(System.getProperty("build.test.dir", "build"));
protected String hostPort = "127.0.0.1:" + PortAssignment.unique();
protected final String hostPort = initHostPort();
protected int maxCnxns = 0;
protected ServerCnxnFactory serverFactory = null;
protected File tmpDir = null;
long initialFdCount;
public ClientBaseWithFixes() {
super();
}
/**
* In general don't use this. Only use in the special case that you
* want to ignore results (for whatever reason) in your test. Don't
@ -154,6 +153,10 @@ public abstract class ClientBaseWithFixes extends ZKTestCase {
private LinkedList<ZooKeeper> allClients;
private boolean allClientsSetup = false;
private RandomAccessFile portNumLockFile;
private File portNumFile;
protected TestableZooKeeper createClient(CountdownWatcher watcher, String hp)
throws IOException, InterruptedException
{
@ -311,6 +314,7 @@ public abstract class ClientBaseWithFixes extends ZKTestCase {
return tmpDir;
}
private static int getPort(String hostPort) {
String[] split = hostPort.split(":");
String portstr = split[split.length-1];
@ -394,6 +398,35 @@ public abstract class ClientBaseWithFixes extends ZKTestCase {
LOG.info("Client test setup finished");
}
private String initHostPort() {
BASETEST.mkdirs();
int port;
for (;;) {
port = PortAssignment.unique();
FileLock lock = null;
portNumLockFile = null;
try {
try {
portNumFile = new File(BASETEST, port + ".lock");
portNumLockFile = new RandomAccessFile(portNumFile, "rw");
try {
lock = portNumLockFile.getChannel().tryLock();
} catch (OverlappingFileLockException e) {
continue;
}
} finally {
if (lock != null)
break;
if (portNumLockFile != null)
portNumLockFile.close();
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return "127.0.0.1:" + port;
}
protected void startServer() throws Exception {
LOG.info("STARTING server");
serverFactory = createNewServerInstance(tmpDir, serverFactory, hostPort, maxCnxns);
@ -434,6 +467,9 @@ public abstract class ClientBaseWithFixes extends ZKTestCase {
stopServer();
portNumLockFile.close();
portNumFile.delete();
if (tmpDir != null) {
Assert.assertTrue("delete " + tmpDir.toString(), recursiveDelete(tmpDir));
}

41
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java
View File

@ -39,8 +39,10 @@ import org.mockito.Mockito;
public class TestSequenceFile extends TestCase {
private static final Log LOG = LogFactory.getLog(TestSequenceFile.class);
private static Configuration conf = new Configuration();
private Configuration conf = new Configuration();
public TestSequenceFile() { }
public TestSequenceFile(String name) { super(name); }
/** Unit tests for SequenceFile. */
@ -126,7 +128,7 @@ public class TestSequenceFile extends TestCase {
}
}
private static void writeTest(FileSystem fs, int count, int seed, Path file,
private void writeTest(FileSystem fs, int count, int seed, Path file,
CompressionType compressionType, CompressionCodec codec)
throws IOException {
fs.delete(file, true);
@ -146,7 +148,7 @@ public class TestSequenceFile extends TestCase {
writer.close();
}
private static void readTest(FileSystem fs, int count, int seed, Path file)
private void readTest(FileSystem fs, int count, int seed, Path file)
throws IOException {
LOG.debug("reading " + count + " records");
SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
@ -202,7 +204,7 @@ public class TestSequenceFile extends TestCase {
}
private static void sortTest(FileSystem fs, int count, int megabytes,
private void sortTest(FileSystem fs, int count, int megabytes,
int factor, boolean fast, Path file)
throws IOException {
fs.delete(new Path(file+".sorted"), true);
@ -212,7 +214,7 @@ public class TestSequenceFile extends TestCase {
LOG.info("done sorting " + count + " debug");
}
private static void checkSort(FileSystem fs, int count, int seed, Path file)
private void checkSort(FileSystem fs, int count, int seed, Path file)
throws IOException {
LOG.info("sorting " + count + " records in memory for debug");
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
@ -249,7 +251,7 @@ public class TestSequenceFile extends TestCase {
LOG.debug("sucessfully checked " + count + " records");
}
private static void mergeTest(FileSystem fs, int count, int seed, Path file,
private void mergeTest(FileSystem fs, int count, int seed, Path file,
CompressionType compressionType,
boolean fast, int factor, int megabytes)
throws IOException {
@ -293,7 +295,7 @@ public class TestSequenceFile extends TestCase {
.merge(sortedNames, file.suffix(".sorted"));
}
private static SequenceFile.Sorter newSorter(FileSystem fs,
private SequenceFile.Sorter newSorter(FileSystem fs,
boolean fast,
int megabytes, int factor) {
SequenceFile.Sorter sorter =
@ -370,7 +372,7 @@ public class TestSequenceFile extends TestCase {
}
private static SequenceFile.Metadata readMetadata(FileSystem fs, Path file)
private SequenceFile.Metadata readMetadata(FileSystem fs, Path file)
throws IOException {
LOG.info("reading file: " + file.toString());
SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
@ -379,7 +381,7 @@ public class TestSequenceFile extends TestCase {
return meta;
}
private static void writeMetadataTest(FileSystem fs, int count, int seed, Path file,
private void writeMetadataTest(FileSystem fs, int count, int seed, Path file,
CompressionType compressionType, CompressionCodec codec, SequenceFile.Metadata metadata)
throws IOException {
fs.delete(file, true);
@ -399,7 +401,7 @@ public class TestSequenceFile extends TestCase {
writer.close();
}
private static void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata)
private void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata)
throws IOException {
fs.delete(sortedFile, true);
LOG.info("sorting: " + unsortedFile + " to: " + sortedFile);
@ -538,7 +540,6 @@ public class TestSequenceFile extends TestCase {
}
public void testRecursiveSeqFileCreate() throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.getLocal(conf);
Path name = new Path(new Path(System.getProperty("test.build.data","."),
"recursiveCreateDir") , "file");
@ -673,7 +674,9 @@ public class TestSequenceFile extends TestCase {
}
}
fs = file.getFileSystem(conf);
TestSequenceFile test = new TestSequenceFile();
fs = file.getFileSystem(test.conf);
LOG.info("count = " + count);
LOG.info("megabytes = " + megabytes);
@ -696,25 +699,25 @@ public class TestSequenceFile extends TestCase {
CompressionType compressionType =
CompressionType.valueOf(compressType);
CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(
conf.getClassByName(compressionCodec),
conf);
test.conf.getClassByName(compressionCodec),
test.conf);
if (rwonly || (create && !merge)) {
writeTest(fs, count, seed, file, compressionType, codec);
readTest(fs, count, seed, file);
test.writeTest(fs, count, seed, file, compressionType, codec);
test.readTest(fs, count, seed, file);
}
if (!rwonly) {
if (merge) {
mergeTest(fs, count, seed, file, compressionType,
test.mergeTest(fs, count, seed, file, compressionType,
fast, factor, megabytes);
} else {
sortTest(fs, count, megabytes, factor, fast, file);
test.sortTest(fs, count, megabytes, factor, fast, file);
}
}
if (check) {
checkSort(fs, count, seed, file);
test.checkSort(fs, count, seed, file);
}
} finally {
fs.close();

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java
View File

@ -46,7 +46,7 @@ public class TestSequenceFileSerialization extends TestCase {
public void testJavaSerialization() throws Exception {
Path file = new Path(System.getProperty("test.build.data",".") +
"/test.seq");
"/testseqser.seq");
fs.delete(file, true);
Writer writer = SequenceFile.createWriter(fs, conf, file, Long.class,

11
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java
View File

@ -64,7 +64,7 @@ public class TestTFileByteArrays {
private String compression = Compression.Algorithm.GZ.getName();
private String comparator = "memcmp";
private String outputFile = "TFileTestByteArrays";
private final String outputFile = getClass().getSimpleName();
/*
* pre-sampled numbers of records in one block, based on the given the
@ -75,17 +75,16 @@ public class TestTFileByteArrays {
private int records1stBlock = usingNative ? 5674 : 4480;
private int records2ndBlock = usingNative ? 5574 : 4263;
public void init(String compression, String comparator, String outputFile,
public void init(String compression, String comparator,
int numRecords1stBlock, int numRecords2ndBlock) {
init(compression, comparator, outputFile);
init(compression, comparator);
this.records1stBlock = numRecords1stBlock;
this.records2ndBlock = numRecords2ndBlock;
}
public void init(String compression, String comparator, String outputFile) {
public void init(String compression, String comparator) {
this.compression = compression;
this.comparator = comparator;
this.outputFile = outputFile;
}
@Before
@ -99,7 +98,7 @@ public class TestTFileByteArrays {
@After
public void tearDown() throws IOException {
if (!skip)
fs.delete(path, true);
fs.delete(path, true);
}
@Test

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileJClassComparatorByteArrays.java
View File

@ -37,8 +37,7 @@ public class TestTFileJClassComparatorByteArrays extends TestTFileByteArrays {
@Override
public void setUp() throws IOException {
init(Compression.Algorithm.GZ.getName(),
"jclass: org.apache.hadoop.io.file.tfile.MyComparator",
"TFileTestJClassComparator");
"jclass: org.apache.hadoop.io.file.tfile.MyComparator");
super.setUp();
}
}

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsByteArrays.java
View File

@ -34,8 +34,7 @@ public class TestTFileLzoCodecsByteArrays extends TestTFileByteArrays {
}
// TODO: sample the generated key/value records, and put the numbers below
init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo",
2605, 2558);
init(Compression.Algorithm.LZO.getName(), "memcmp", 2605, 2558);
if (!skip)
super.setUp();
}

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsStreams.java
View File

@ -32,7 +32,7 @@ public class TestTFileLzoCodecsStreams extends TestTFileStreams {
if (skip) {
System.out.println("Skipped");
}
init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo");
init(Compression.Algorithm.LZO.getName(), "memcmp");
if (!skip)
super.setUp();
}

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsByteArrays.java
View File

@ -25,8 +25,7 @@ public class TestTFileNoneCodecsByteArrays extends TestTFileByteArrays {
*/
@Override
public void setUp() throws IOException {
init(Compression.Algorithm.NONE.getName(), "memcmp", "TFileTestCodecsNone",
24, 24);
init(Compression.Algorithm.NONE.getName(), "memcmp", 24, 24);
super.setUp();
}
}

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsJClassComparatorByteArrays.java
View File

@ -33,8 +33,7 @@ public class TestTFileNoneCodecsJClassComparatorByteArrays extends TestTFileByte
@Override
public void setUp() throws IOException {
init(Compression.Algorithm.NONE.getName(),
"jclass: org.apache.hadoop.io.file.tfile.MyComparator",
"TestTFileNoneCodecsJClassComparatorByteArrays", 24, 24);
"jclass: org.apache.hadoop.io.file.tfile.MyComparator", 24, 24);
super.setUp();
}
}

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsStreams.java
View File

@ -26,7 +26,7 @@ public class TestTFileNoneCodecsStreams extends TestTFileStreams {
*/
@Override
public void setUp() throws IOException {
init(Compression.Algorithm.NONE.getName(), "memcmp", "TFileTestCodecsNone");
init(Compression.Algorithm.NONE.getName(), "memcmp");
super.setUp();
}
}

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java
View File

@ -57,12 +57,11 @@ public class TestTFileStreams extends TestCase {
private String compression = Compression.Algorithm.GZ.getName();
private String comparator = "memcmp";
private String outputFile = "TFileTestStreams";
private final String outputFile = getClass().getSimpleName();
public void init(String compression, String comparator, String outputFile) {
public void init(String compression, String comparator) {
this.compression = compression;
this.comparator = comparator;
this.outputFile = outputFile;
}
@Override

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
View File

@ -64,6 +64,9 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@RunWith(MockitoJUnitRunner.class)
public class TestMetricsSystemImpl {
private static final Log LOG = LogFactory.getLog(TestMetricsSystemImpl.class);
static { DefaultMetricsSystem.setMiniClusterMode(true); }
@Captor private ArgumentCaptor<MetricsRecord> r1;
@Captor private ArgumentCaptor<MetricsRecord> r2;
private static String hostname = MetricsSystemImpl.getHostname();

1
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsCreateMkdir.java
View File

@ -41,7 +41,6 @@ public class TestFcHdfsCreateMkdir extends
@BeforeClass
public static void clusterSetupAtBegining()
throws IOException, LoginException, URISyntaxException {
FileContextTestHelper.TEST_ROOT_DIR = "/tmp/TestFcHdfsCreateMkdir";
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
fc = FileContext.getFileContext(cluster.getURI(0), conf);

14
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsPermission.java
View File

@ -35,13 +35,25 @@ import org.junit.BeforeClass;
public class TestFcHdfsPermission extends FileContextPermissionBase {
private static final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
private static FileContext fc;
private static MiniDFSCluster cluster;
private static Path defaultWorkingDirectory;
@Override
protected FileContextTestHelper getFileContextHelper() {
return fileContextTestHelper;
}
@Override
protected FileContext getFileContext() {
return fc;
}
@BeforeClass
public static void clusterSetupAtBegining()
throws IOException, LoginException, URISyntaxException {
FileContextTestHelper.TEST_ROOT_DIR = "/tmp/TestFcHdfsPermission";
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
fc = FileContext.getFileContext(cluster.getURI(0), conf);

18
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSetUMask.java
View File

@ -42,6 +42,7 @@ import org.junit.Test;
public class TestFcHdfsSetUMask {
private static FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
private static MiniDFSCluster cluster;
private static Path defaultWorkingDirectory;
private static FileContext fc;
@ -82,7 +83,6 @@ public class TestFcHdfsSetUMask {
@BeforeClass
public static void clusterSetupAtBegining()
throws IOException, LoginException, URISyntaxException {
FileContextTestHelper.TEST_ROOT_DIR = "/tmp/TestFcHdfsSetUMask";
Configuration conf = new HdfsConfiguration();
// set permissions very restrictive
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
@ -111,12 +111,12 @@ public class TestFcHdfsSetUMask {
@Before
public void setUp() throws Exception {
fc.setUMask(WIDE_OPEN_TEST_UMASK);
fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DEFAULT_PERM, true);
}
@After
public void tearDown() throws Exception {
fc.delete(getTestRootPath(fc), true);
fc.delete(fileContextTestHelper.getTestRootPath(fc), true);
}
@Test
@ -200,7 +200,7 @@ public class TestFcHdfsSetUMask {
public void testMkdirWithExistingDir(FsPermission umask,
FsPermission expectedPerms) throws IOException {
Path f = getTestRootPath(fc, "aDir");
Path f = fileContextTestHelper.getTestRootPath(fc, "aDir");
fc.setUMask(umask);
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
Assert.assertTrue(isDir(fc, f));
@ -211,13 +211,13 @@ public class TestFcHdfsSetUMask {
public void testMkdirRecursiveWithNonExistingDir(FsPermission umask,
FsPermission expectedPerms, FsPermission expectedParentPerms)
throws IOException {
Path f = getTestRootPath(fc, "NonExistant2/aDir");
Path f = fileContextTestHelper.getTestRootPath(fc, "NonExistant2/aDir");
fc.setUMask(umask);
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
Assert.assertTrue(isDir(fc, f));
Assert.assertEquals("permissions on directory are wrong",
expectedPerms, fc.getFileStatus(f).getPermission());
Path fParent = getTestRootPath(fc, "NonExistant2");
Path fParent = fileContextTestHelper.getTestRootPath(fc, "NonExistant2");
Assert.assertEquals("permissions on parent directory are wrong",
expectedParentPerms, fc.getFileStatus(fParent).getPermission());
}
@ -225,7 +225,7 @@ public class TestFcHdfsSetUMask {
public void testCreateRecursiveWithExistingDir(FsPermission umask,
FsPermission expectedPerms) throws IOException {
Path f = getTestRootPath(fc,"foo");
Path f = fileContextTestHelper.getTestRootPath(fc,"foo");
fc.setUMask(umask);
createFile(fc, f);
Assert.assertTrue(isFile(fc, f));
@ -237,8 +237,8 @@ public class TestFcHdfsSetUMask {
public void testCreateRecursiveWithNonExistingDir(FsPermission umask,
FsPermission expectedDirPerms, FsPermission expectedFilePerms)
throws IOException {
Path f = getTestRootPath(fc,"NonExisting/foo");
Path fParent = getTestRootPath(fc, "NonExisting");
Path f = fileContextTestHelper.getTestRootPath(fc,"NonExisting/foo");
Path fParent = fileContextTestHelper.getTestRootPath(fc, "NonExisting");
Assert.assertFalse(exists(fc, fParent));
fc.setUMask(umask);
createFile(fc, f);

14
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.hadoop.fs;
import static org.apache.hadoop.fs.FileContextTestHelper.getAbsoluteTestRootDir;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@ -51,10 +50,10 @@ public class TestFcHdfsSymlink extends FileContextSymlinkBaseTest {
((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
}
private static FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
private static MiniDFSCluster cluster;
private static WebHdfsFileSystem webhdfs;
private static DistributedFileSystem dfs;
@Override
protected String getScheme() {
@ -86,7 +85,6 @@ public class TestFcHdfsSymlink extends FileContextSymlinkBaseTest {
@BeforeClass
public static void testSetUp() throws Exception {
FileContextTestHelper.TEST_ROOT_DIR = "/tmp/TestFcHdfsSymlink";
Configuration conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
conf.set(FsPermission.UMASK_LABEL, "000");
@ -104,10 +102,10 @@ public class TestFcHdfsSymlink extends FileContextSymlinkBaseTest {
@Test
/** Access a file using a link that spans Hdfs to LocalFs */
public void testLinkAcrossFileSystems() throws IOException {
Path localDir = new Path("file://"+getAbsoluteTestRootDir(fc)+"/test");
Path localFile = new Path("file://"+getAbsoluteTestRootDir(fc)+"/test/file");
Path link = new Path(testBaseDir1(), "linkToFile");
FileContext localFc = FileContext.getLocalFSFileContext();
Path localDir = new Path("file://"+fileContextTestHelper.getAbsoluteTestRootDir(localFc)+"/test");
Path localFile = new Path("file://"+fileContextTestHelper.getAbsoluteTestRootDir(localFc)+"/test/file");
Path link = new Path(testBaseDir1(), "linkToFile");
localFc.delete(localDir, true);
localFc.mkdir(localDir, FileContext.DEFAULT_PERM, true);
localFc.setWorkingDirectory(localDir);
@ -121,12 +119,12 @@ public class TestFcHdfsSymlink extends FileContextSymlinkBaseTest {
@Test
/** Test renaming a file across two file systems using a link */
public void testRenameAcrossFileSystemsViaLink() throws IOException {
Path localDir = new Path("file://"+getAbsoluteTestRootDir(fc)+"/test");
FileContext localFc = FileContext.getLocalFSFileContext();
Path localDir = new Path("file://"+fileContextTestHelper.getAbsoluteTestRootDir(localFc)+"/test");
Path hdfsFile = new Path(testBaseDir1(), "file");
Path link = new Path(testBaseDir1(), "link");
Path hdfsFileNew = new Path(testBaseDir1(), "fileNew");
Path hdfsFileNewViaLink = new Path(link, "fileNew");
FileContext localFc = FileContext.getLocalFSFileContext();
localFc.delete(localDir, true);
localFc.mkdir(localDir, FileContext.DEFAULT_PERM, true);
localFc.setWorkingDirectory(localDir);

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.fs;
import static org.apache.hadoop.fs.FileContextTestHelper.exists;
import static org.apache.hadoop.fs.FileContextTestHelper.getTestRootPath;
import java.io.IOException;
import java.net.URISyntaxException;
@ -42,6 +41,7 @@ import org.junit.Test;
public class TestHDFSFileContextMainOperations extends
FileContextMainOperationsBaseTest {
private static FileContextTestHelper fileContextTestHelper = new FileContextTestHelper();
private static MiniDFSCluster cluster;
private static Path defaultWorkingDirectory;
private static HdfsConfiguration CONF = new HdfsConfiguration();
@ -49,8 +49,6 @@ public class TestHDFSFileContextMainOperations extends
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
FileContextTestHelper.TEST_ROOT_DIR =
"/tmp/TestHDFSFileContextMainOperations";
cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(2).build();
cluster.waitClusterUp();
fc = FileContext.getFileContext(cluster.getURI(0), CONF);
@ -103,6 +101,10 @@ public class TestHDFSFileContextMainOperations extends
return e;
}
private Path getTestRootPath(FileContext fc, String path) {
return fileContextTestHelper.getTestRootPath(fc, path);
}
@Test
public void testOldRenameWithQuota() throws Exception {
DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem();

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java
View File

@ -43,10 +43,14 @@ public class TestViewFileSystemAtHdfsRoot extends ViewFileSystemBaseTest {
private static Configuration CONF = new Configuration();
private static FileSystem fHdfs;
@Override
protected FileSystemTestHelper createFileSystemHelper() {
return new FileSystemTestHelper("/tmp/TestViewFileSystemAtHdfsRoot");
}
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
FileSystemTestHelper.TEST_ROOT_DIR = "/tmp/TestViewFileSystemAtHdfsRoot";
SupportsBlocks = true;
CONF.setBoolean(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
View File

@ -48,10 +48,14 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
private FileSystem fsTarget2;
Path targetTestRoot2;
@Override
protected FileSystemTestHelper createFileSystemHelper() {
return new FileSystemTestHelper("/tmp/TestViewFileSystemHdfs");
}
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
FileSystemTestHelper.TEST_ROOT_DIR = "/tmp/TestViewFileSystemHdfs";
SupportsBlocks = true;
CONF.setBoolean(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
@ -87,7 +91,7 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
// create the test root on local_fs
fsTarget = fHdfs;
fsTarget2 = fHdfs2;
targetTestRoot2 = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2);
targetTestRoot2 = new FileSystemTestHelper().getAbsoluteTestRootPath(fsTarget2);
super.setUp();
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsAtHdfsRoot.java
View File

@ -23,7 +23,6 @@ import java.net.URISyntaxException;
import javax.security.auth.login.LoginException;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestHelper;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
@ -47,7 +46,6 @@ public class TestViewFsAtHdfsRoot extends ViewFsBaseTest {
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
FileContextTestHelper.TEST_ROOT_DIR = "/tmp/TestViewFsAtHdfsRoot";
SupportsBlocks = true;
CONF.setBoolean(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
View File

@ -61,6 +61,7 @@ public class TestViewFsDefaultValue {
static final String testFileDir = "/tmp/test/";
static final String testFileName = testFileDir + "testFileStatusSerialziation";
private static MiniDFSCluster cluster;
private static final FileSystemTestHelper fileSystemTestHelper = new FileSystemTestHelper();
private static Configuration CONF = new Configuration();
private static FileSystem fHdfs;
private static FileSystem vfs;
@ -81,7 +82,7 @@ public class TestViewFsDefaultValue {
cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(DFS_REPLICATION_DEFAULT + 1).build();
cluster.waitClusterUp();
fHdfs = cluster.getFileSystem();
FileSystemTestHelper.createFile(fHdfs, testFileName);
fileSystemTestHelper.createFile(fHdfs, testFileName);
Configuration conf = ViewFileSystemTestSetup.createConfig();
ConfigUtil.addLink(conf, "/tmp", new URI(fHdfs.getUri().toString() +
"/tmp"));

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java
View File

@ -52,6 +52,7 @@ public class TestViewFsFileStatusHdfs {
static final String testfilename = "/tmp/testFileStatusSerialziation";
static final String someFile = "/hdfstmp/someFileForTestGetFileChecksum";
private static final FileSystemTestHelper fileSystemTestHelper = new FileSystemTestHelper();
private static MiniDFSCluster cluster;
private static Path defaultWorkingDirectory;
private static Configuration CONF = new Configuration();
@ -79,7 +80,7 @@ public class TestViewFsFileStatusHdfs {
@Test
public void testFileStatusSerialziation()
throws IOException, URISyntaxException {
long len = FileSystemTestHelper.createFile(fHdfs, testfilename);
long len = fileSystemTestHelper.createFile(fHdfs, testfilename);
FileStatus stat = vfs.getFileStatus(new Path(testfilename));
assertEquals(len, stat.getLen());
// check serialization/deserialization
@ -95,8 +96,8 @@ public class TestViewFsFileStatusHdfs {
@Test
public void testGetFileChecksum() throws IOException, URISyntaxException {
// Create two different files in HDFS
FileSystemTestHelper.createFile(fHdfs, someFile);
FileSystemTestHelper.createFile(fHdfs, FileSystemTestHelper
fileSystemTestHelper.createFile(fHdfs, someFile);
fileSystemTestHelper.createFile(fHdfs, fileSystemTestHelper
.getTestRootPath(fHdfs, someFile + "other"), 1, 512);
// Get checksum through ViewFS
FileChecksum viewFSCheckSum = vfs.getFileChecksum(

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsHdfs.java
View File

@ -24,7 +24,6 @@ import java.net.URISyntaxException;
import javax.security.auth.login.LoginException;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestHelper;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@ -43,7 +42,6 @@ public class TestViewFsHdfs extends ViewFsBaseTest {
@BeforeClass
public static void clusterSetupAtBegining() throws IOException,
LoginException, URISyntaxException {
FileContextTestHelper.TEST_ROOT_DIR = "/tmp/TestViewFsHdfs";
SupportsBlocks = true;
CONF.setBoolean(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);

19
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java
View File

@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hdfs.web;
import static org.apache.hadoop.fs.FileSystemTestHelper.exists;
import static org.apache.hadoop.fs.FileSystemTestHelper.getTestRootPath;
import java.io.IOException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
@ -29,7 +26,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSMainOperationsBaseTest;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@ -53,13 +49,15 @@ public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest {
private static MiniDFSCluster cluster = null;
private static Path defaultWorkingDirectory;
private static FileSystem fileSystem;
@Override
protected FileSystem createFileSystem() throws Exception {
return fileSystem;
}
@BeforeClass
public static void setupCluster() {
// Initialize the test root directory to a DFS like path
// since we are testing based on the MiniDFSCluster.
FileSystemTestHelper.TEST_ROOT_DIR = "/tmp/TestFSMainOperationsWebHdfs";
final Configuration conf = new Configuration();
conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
@ -78,14 +76,14 @@ public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest {
final UserGroupInformation current = UserGroupInformation.getCurrentUser();
final UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
current.getShortUserName() + "x", new String[]{"user"});
fSys = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
fileSystem = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
@Override
public FileSystem run() throws Exception {
return FileSystem.get(new URI(uri), conf);
}
});
defaultWorkingDirectory = fSys.getWorkingDirectory();
defaultWorkingDirectory = fileSystem.getWorkingDirectory();
} catch (Exception e) {
throw new RuntimeException(e);
}
@ -164,4 +162,5 @@ public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest {
// also okay for HDFS.
}
}
}

21
hadoop-project/pom.xml
View File

@ -36,6 +36,9 @@
<test.exclude>_</test.exclude>
<test.exclude.pattern>_</test.exclude.pattern>
<!-- number of threads/forks to use when running tests in parallel, see parallel-tests profile -->
<testsThreadCount>4</testsThreadCount>
<!-- platform encoding override -->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@ -967,5 +970,23 @@
</plugins>
</build>
</profile>
<!-- Copied into specific modules supporting parallel testing. Will be uncommented as soon as all modules support this.
<profile>
<id>parallel-tests</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkMode>perthread</forkMode>
<threadCount>${testsThreadCount}</threadCount>
<parallel>classes</parallel>
</configuration>
</plugin>
</plugins>
</build>
</profile>
-->
</profiles>
</project>

6
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
View File

@ -56,14 +56,14 @@ public class TestDistCpViewFs {
public static void setup() throws URISyntaxException{
try {
Path fswd = FileSystem.get(getConf()).getWorkingDirectory();
Configuration vConf = ViewFileSystemTestSetup.createConfig();
ConfigUtil.addLink(vConf, "/usr", new URI(fswd.toString()));
Configuration vConf = ViewFileSystemTestSetup.createConfig(false);
ConfigUtil.addLink(vConf, "/usr", new URI(fswd.toString()));
fs = FileSystem.get(FsConstants.VIEWFS_URI, vConf);
fs.setWorkingDirectory(new Path("/usr"));
listFile = new Path("target/tmp/listing").makeQualified(fs.getUri(),
fs.getWorkingDirectory());
target = new Path("target/tmp/target").makeQualified(fs.getUri(),
fs.getWorkingDirectory());
fs.getWorkingDirectory());
root = new Path("target/tmp").makeQualified(fs.getUri(),
fs.getWorkingDirectory()).toString();
TestDistCpUtils.delete(fs, root);