From eb7af141e4b63831bf36edff2c9c38c2fdbe76af Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Tue, 4 Jun 2013 00:29:37 +0000 Subject: [PATCH] HADOOP-9287. Parallel-testing hadoop-common. Contributed by Andrey Klochkov git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1489250 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + hadoop-common-project/hadoop-common/pom.xml | 17 +++ .../apache/hadoop/conf/TestConfiguration.java | 5 +- .../conf/TestConfigurationDeprecation.java | 8 +- .../hadoop/fs/FCStatisticsBaseTest.java | 4 +- .../hadoop/fs/FSMainOperationsBaseTest.java | 79 +++++++------ .../fs/FileContextCreateMkdirBaseTest.java | 19 +++- .../fs/FileContextMainOperationsBaseTest.java | 38 ++++--- .../hadoop/fs/FileContextPermissionBase.java | 23 ++-- .../hadoop/fs/FileContextSymlinkBaseTest.java | 1 + .../hadoop/fs/FileContextTestHelper.java | 44 ++++---- .../apache/hadoop/fs/FileContextUtilBase.java | 14 +-- .../hadoop/fs/FileSystemTestHelper.java | 57 ++++++---- .../TestFSMainOperationsLocalFileSystem.java | 9 +- .../hadoop/fs/TestFcLocalFsPermission.java | 6 +- .../fs/TestFileContextDeleteOnExit.java | 9 +- .../TestLocalFSFileContextMainOperations.java | 2 +- .../fs/TestLocalFSFileContextSymlink.java | 5 +- .../apache/hadoop/fs/TestLocalFileSystem.java | 6 +- .../hadoop/fs/TestLocalFsFCStatistics.java | 6 +- .../hadoop/fs/shell/TestCopyPreserveFlag.java | 12 +- .../fs/viewfs/TestChRootedFileSystem.java | 30 ++--- .../hadoop/fs/viewfs/TestChRootedFs.java | 37 +++--- .../TestFSMainOperationsLocalFileSystem.java | 17 +-- .../fs/viewfs/TestFcCreateMkdirLocalFs.java | 4 +- .../viewfs/TestFcMainOperationsLocalFs.java | 4 +- .../fs/viewfs/TestFcPermissionsLocalFs.java | 10 +- .../viewfs/TestViewFileSystemDelegation.java | 2 +- .../TestViewFileSystemLocalFileSystem.java | 3 +- ...ileSystemWithAuthorityLocalFileSystem.java | 2 +- .../hadoop/fs/viewfs/TestViewFsTrash.java | 13 ++- .../fs/viewfs/ViewFileSystemBaseTest.java | 106 ++++++++++-------- .../fs/viewfs/ViewFileSystemTestSetup.java | 17 ++- .../hadoop/fs/viewfs/ViewFsBaseTest.java | 69 ++++++------ .../hadoop/fs/viewfs/ViewFsTestSetup.java | 10 +- .../apache/hadoop/ha/ClientBaseWithFixes.java | 46 +++++++- .../apache/hadoop/io/TestSequenceFile.java | 41 +++---- .../io/TestSequenceFileSerialization.java | 2 +- .../io/file/tfile/TestTFileByteArrays.java | 11 +- .../TestTFileJClassComparatorByteArrays.java | 3 +- .../tfile/TestTFileLzoCodecsByteArrays.java | 3 +- .../file/tfile/TestTFileLzoCodecsStreams.java | 2 +- .../tfile/TestTFileNoneCodecsByteArrays.java | 3 +- ...eNoneCodecsJClassComparatorByteArrays.java | 3 +- .../tfile/TestTFileNoneCodecsStreams.java | 2 +- .../io/file/tfile/TestTFileStreams.java | 5 +- .../metrics2/impl/TestMetricsSystemImpl.java | 3 + .../hadoop/fs/TestFcHdfsPermission.java | 13 +++ .../apache/hadoop/fs/TestFcHdfsSetUMask.java | 17 +-- .../apache/hadoop/fs/TestFcHdfsSymlink.java | 13 +-- .../fs/TestHDFSFileContextMainOperations.java | 6 +- .../viewfs/TestViewFileSystemAtHdfsRoot.java | 6 + .../fs/viewfs/TestViewFileSystemHdfs.java | 7 +- .../fs/viewfs/TestViewFsFileStatusHdfs.java | 7 +- .../hdfs/web/TestFSMainOperationsWebHdfs.java | 15 ++- hadoop-project/pom.xml | 21 ++++ .../apache/hadoop/tools/TestDistCpViewFs.java | 6 +- 57 files changed, 562 insertions(+), 363 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 25cf12bd83b..2bc8cd3c648 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -95,6 +95,8 @@ Release 2.1.0-beta - UNRELEASED HADOOP-7391 Document Interface Classification from HADOOP-5073 (sanjay Radia) + HADOOP-9287. Parallel-testing hadoop-common (Andrey Klochkov via jlowe) + OPTIMIZATIONS HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 03bec1da35e..b14e7eb42f2 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -653,5 +653,22 @@ + + parallel-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + + perthread + ${testsThreadCount} + classes + -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -DminiClusterDedicatedDirs=true + + + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 1bd791e41cc..af2e3e641dd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -39,6 +39,7 @@ import java.util.regex.Pattern; import junit.framework.TestCase; import static org.junit.Assert.assertArrayEquals; + import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration.IntegerRanges; import org.apache.hadoop.fs.Path; @@ -49,8 +50,8 @@ import org.codehaus.jackson.map.ObjectMapper; public class TestConfiguration extends TestCase { private Configuration conf; - final static String CONFIG = new File("./test-config.xml").getAbsolutePath(); - final static String CONFIG2 = new File("./test-config2.xml").getAbsolutePath(); + final static String CONFIG = new File("./test-config-TestConfiguration.xml").getAbsolutePath(); + final static String CONFIG2 = new File("./test-config2-TestConfiguration.xml").getAbsolutePath(); final static Random RAN = new Random(); final static String XMLHEADER = IBM_JAVA?"": diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java index 014844e28bc..6d74be2fda6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java @@ -36,11 +36,9 @@ import org.junit.Test; public class TestConfigurationDeprecation { private Configuration conf; - final static String CONFIG = new File("./test-config.xml").getAbsolutePath(); - final static String CONFIG2 = - new File("./test-config2.xml").getAbsolutePath(); - final static String CONFIG3 = - new File("./test-config3.xml").getAbsolutePath(); + final static String CONFIG = new File("./test-config-TestConfigurationDeprecation.xml").getAbsolutePath(); + final static String CONFIG2 = new File("./test-config2-TestConfigurationDeprecation.xml").getAbsolutePath(); + final static String CONFIG3 = new File("./test-config3-TestConfigurationDeprecation.xml").getAbsolutePath(); BufferedWriter out; static { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java index 38d99f01e7e..2718291f6c2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java @@ -39,6 +39,8 @@ public abstract class FCStatisticsBaseTest { static protected int blockSize = 512; static protected int numBlocks = 1; + protected final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); + //fc should be set appropriately by the deriving test. protected static FileContext fc = null; @@ -47,7 +49,7 @@ public abstract class FCStatisticsBaseTest { URI fsUri = getFsUri(); Statistics stats = FileContext.getStatistics(fsUri); Assert.assertEquals(0, stats.getBytesRead()); - Path filePath = getTestRootPath(fc, "file1"); + Path filePath = fileContextTestHelper .getTestRootPath(fc, "file1"); createFile(fc, filePath, numBlocks, blockSize); Assert.assertEquals(0, stats.getBytesRead()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java index e9677badc38..8506ea9bee8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java @@ -34,8 +34,6 @@ import org.junit.Before; import org.junit.Test; import org.mortbay.log.Log; -import static org.apache.hadoop.fs.FileSystemTestHelper.*; - /** *

* A collection of tests for the {@link FileSystem}. @@ -55,7 +53,7 @@ import static org.apache.hadoop.fs.FileSystemTestHelper.*; * @AfterClass public static void ClusterShutdownAtEnd() *

*/ -public abstract class FSMainOperationsBaseTest { +public abstract class FSMainOperationsBaseTest extends FileSystemTestHelper { private static String TEST_DIR_AAA2 = "test/hadoop2/aaa"; private static String TEST_DIR_AAA = "test/hadoop/aaa"; @@ -63,8 +61,8 @@ public abstract class FSMainOperationsBaseTest { private static String TEST_DIR_AXX = "test/hadoop/axx"; private static int numBlocks = 2; - - protected static FileSystem fSys; + + protected FileSystem fSys; final private static PathFilter DEFAULT_FILTER = new PathFilter() { @Override @@ -87,8 +85,18 @@ public abstract class FSMainOperationsBaseTest { protected static final byte[] data = getFileData(numBlocks, getDefaultBlockSize()); + abstract protected FileSystem createFileSystem() throws Exception; + + public FSMainOperationsBaseTest() { + } + + public FSMainOperationsBaseTest(String testRootDir) { + super(testRootDir); + } + @Before public void setUp() throws Exception { + fSys = createFileSystem(); fSys.mkdirs(getTestRootPath(fSys, "test")); } @@ -157,7 +165,7 @@ public abstract class FSMainOperationsBaseTest { // Now open a file relative to the wd we just set above. Path absolutePath = new Path(absoluteDir, "foo"); - FileSystemTestHelper.createFile(fSys, absolutePath); + createFile(fSys, absolutePath); fSys.open(new Path("foo")).close(); @@ -308,11 +316,11 @@ public abstract class FSMainOperationsBaseTest { paths = fSys.listStatus(getTestRootPath(fSys, "test/hadoop")); Assert.assertEquals(3, paths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/a"), + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, "test/hadoop/a"), paths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/b"), + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, "test/hadoop/b"), paths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, "test/hadoop/c"), + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, "test/hadoop/c"), paths)); paths = fSys.listStatus(getTestRootPath(fSys, "test/hadoop/a")); @@ -359,9 +367,9 @@ public abstract class FSMainOperationsBaseTest { FileStatus[] filteredPaths = fSys.listStatus( getTestRootPath(fSys, "test/hadoop"), TEST_X_FILTER); Assert.assertEquals(2,filteredPaths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA), filteredPaths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX), filteredPaths)); } @@ -417,9 +425,9 @@ public abstract class FSMainOperationsBaseTest { FileStatus[] paths = fSys.globStatus( getTestRootPath(fSys, "test/hadoop*")); Assert.assertEquals(2, paths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, "test/hadoop"), paths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, "test/hadoop2"), paths)); } @@ -442,10 +450,10 @@ public abstract class FSMainOperationsBaseTest { FileStatus[] paths = fSys.globStatus( getTestRootPath(fSys, "test/hadoop*/*")); Assert.assertEquals(4, paths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA), paths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA), paths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX), paths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA2), paths)); + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AAA), paths)); + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA), paths)); + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX), paths)); + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AAA2), paths)); } @Test @@ -466,9 +474,9 @@ public abstract class FSMainOperationsBaseTest { FileStatus[] paths = fSys.globStatus( getTestRootPath(fSys, "test/hadoop/ax?")); Assert.assertEquals(2, paths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA), paths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX), paths)); } @@ -513,11 +521,11 @@ public abstract class FSMainOperationsBaseTest { getTestRootPath(fSys, "test/hadoop/*"), DEFAULT_FILTER); Assert.assertEquals(3, filteredPaths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AAA), filteredPaths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA), filteredPaths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX), filteredPaths)); } @@ -541,11 +549,11 @@ public abstract class FSMainOperationsBaseTest { getTestRootPath(fSys, "test/hadoop/a??"), DEFAULT_FILTER); Assert.assertEquals(3, filteredPaths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AAA), + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AAA), filteredPaths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA), + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA), filteredPaths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX), + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX), filteredPaths)); } @@ -569,9 +577,9 @@ public abstract class FSMainOperationsBaseTest { getTestRootPath(fSys, "test/hadoop/*"), TEST_X_FILTER); Assert.assertEquals(2, filteredPaths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA), filteredPaths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX), filteredPaths)); } @@ -617,9 +625,9 @@ public abstract class FSMainOperationsBaseTest { getTestRootPath(fSys, "test/hadoop/a??"), TEST_X_FILTER); Assert.assertEquals(2, filteredPaths.length); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXA), + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXA), filteredPaths)); - Assert.assertTrue(containsPath(getTestRootPath(fSys, TEST_DIR_AXX), + Assert.assertTrue(containsTestRootPath(getTestRootPath(fSys, TEST_DIR_AXX), filteredPaths)); } @@ -1090,9 +1098,9 @@ public abstract class FSMainOperationsBaseTest { public void testCopyToLocalWithUseRawLocalFileSystemOption() throws Exception { Configuration conf = new Configuration(); FileSystem fSys = new RawLocalFileSystem(); - Path fileToFS = new Path(TEST_ROOT_DIR, "fs.txt"); - Path fileToLFS = new Path(TEST_ROOT_DIR, "test.txt"); - Path crcFileAtLFS = new Path(TEST_ROOT_DIR, ".test.txt.crc"); + Path fileToFS = new Path(getTestRootDir(), "fs.txt"); + Path fileToLFS = new Path(getTestRootDir(), "test.txt"); + Path crcFileAtLFS = new Path(getTestRootDir(), ".test.txt.crc"); fSys.initialize(new URI("file:///"), conf); writeFile(fSys, fileToFS); if (fSys.exists(crcFileAtLFS)) @@ -1112,7 +1120,7 @@ public abstract class FSMainOperationsBaseTest { } protected void createFile(Path path) throws IOException { - FileSystemTestHelper.createFile(fSys, path); + createFile(fSys, path); } @SuppressWarnings("deprecation") @@ -1125,10 +1133,11 @@ public abstract class FSMainOperationsBaseTest { Assert.assertEquals("Source exists", srcExists, exists(fSys, src)); Assert.assertEquals("Destination exists", dstExists, exists(fSys, dst)); } - private boolean containsPath(Path path, FileStatus[] filteredPaths) + private boolean containsTestRootPath(Path path, FileStatus[] filteredPaths) throws IOException { + Path testRootPath = getTestRootPath(fSys, path.toString()); for(int i = 0; i < filteredPaths.length; i ++) { - if (getTestRootPath(fSys, path.toString()).equals( + if (testRootPath.equals( filteredPaths[i].getPath())) return true; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java index a6878a06e0e..bd1f0aedadb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java @@ -50,7 +50,8 @@ import org.apache.commons.logging.impl.Log4JLogger; */ public abstract class FileContextCreateMkdirBaseTest { - + + protected final FileContextTestHelper fileContextTestHelper; protected static FileContext fc; { @@ -63,6 +64,13 @@ public abstract class FileContextCreateMkdirBaseTest { } } + public FileContextCreateMkdirBaseTest() { + fileContextTestHelper = createFileContextHelper(); + } + + protected FileContextTestHelper createFileContextHelper() { + return new FileContextTestHelper(); + } @Before public void setUp() throws Exception { @@ -149,4 +157,13 @@ public abstract class FileContextCreateMkdirBaseTest { createFile(fc, f); Assert.assertTrue(isFile(fc, f)); } + + private Path getTestRootPath(FileContext fc) { + return fileContextTestHelper.getTestRootPath(fc); + } + + private Path getTestRootPath(FileContext fc, String pathString) { + return fileContextTestHelper.getTestRootPath(fc, pathString); + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index d6efc52c42a..8f74b460507 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -23,6 +23,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.EnumSet; +import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.Rename; @@ -61,17 +62,10 @@ public abstract class FileContextMainOperationsBaseTest { private static String TEST_DIR_AXA = "test/hadoop/axa"; private static String TEST_DIR_AXX = "test/hadoop/axx"; private static int numBlocks = 2; - - public static final Path LOCAL_FS_ROOT_PATH; - - static { - File testBuildData = new File(System.getProperty("test.build.data", - "build/test/data")); - Path localFsRootPath = new Path(testBuildData.getAbsolutePath(), - "root-uri"); - LOCAL_FS_ROOT_PATH = localFsRootPath.makeQualified(LocalFileSystem.NAME, null); - } - + + public Path localFsRootPath; + + protected final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); protected static FileContext fc; final private static PathFilter DEFAULT_FILTER = new PathFilter() { @@ -97,13 +91,18 @@ public abstract class FileContextMainOperationsBaseTest { @Before public void setUp() throws Exception { + File testBuildData = new File(System.getProperty("test.build.data", + "build/test/data"), RandomStringUtils.randomAlphanumeric(10)); + Path rootPath = new Path(testBuildData.getAbsolutePath(), + "root-uri"); + localFsRootPath = rootPath.makeQualified(LocalFileSystem.NAME, null); fc.mkdir(getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true); } @After public void tearDown() throws Exception { - fc.delete(new Path(getAbsoluteTestRootPath(fc), new Path("test")), true); - fc.delete(LOCAL_FS_ROOT_PATH, true); + fc.delete(new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")), true); + fc.delete(localFsRootPath, true); } @@ -136,7 +135,7 @@ public abstract class FileContextMainOperationsBaseTest { public void testWorkingDirectory() throws Exception { // First we cd to our test root - Path workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test")); + Path workDir = new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")); fc.setWorkingDirectory(workDir); Assert.assertEquals(workDir, fc.getWorkingDirectory()); @@ -149,7 +148,7 @@ public abstract class FileContextMainOperationsBaseTest { // cd using a relative path // Go back to our test root - workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test")); + workDir = new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")); fc.setWorkingDirectory(workDir); Assert.assertEquals(workDir, fc.getWorkingDirectory()); @@ -184,7 +183,7 @@ public abstract class FileContextMainOperationsBaseTest { // Try a URI - absoluteDir = new Path(LOCAL_FS_ROOT_PATH, "existingDir"); + absoluteDir = new Path(localFsRootPath, "existingDir"); fc.mkdir(absoluteDir, FileContext.DEFAULT_PERM, true); fc.setWorkingDirectory(absoluteDir); Assert.assertEquals(absoluteDir, fc.getWorkingDirectory()); @@ -1181,6 +1180,7 @@ public abstract class FileContextMainOperationsBaseTest { Assert.assertEquals("Source exists", srcExists, exists(fc, src)); Assert.assertEquals("Destination exists", dstExists, exists(fc, dst)); } + private boolean containsPath(Path path, FileStatus[] filteredPaths) throws IOException { for(int i = 0; i < filteredPaths.length; i ++) { @@ -1189,5 +1189,9 @@ public abstract class FileContextMainOperationsBaseTest { return true; } return false; - } + } + + private Path getTestRootPath(FileContext fc, String pathString) { + return fileContextTestHelper.getTestRootPath(fc, pathString); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java index 2d7c687c6dd..6ceae962887 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java @@ -68,16 +68,25 @@ public abstract class FileContextPermissionBase { } } - protected static FileContext fc; + protected FileContextTestHelper fileContextTestHelper; + protected FileContext fc; + protected FileContextTestHelper getFileContextHelper() { + return new FileContextTestHelper(); + } + + protected abstract FileContext getFileContext() throws Exception; + @Before public void setUp() throws Exception { - fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true); + fileContextTestHelper = getFileContextHelper(); + fc = getFileContext(); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DEFAULT_PERM, true); } @After public void tearDown() throws Exception { - fc.delete(getTestRootPath(fc), true); + fc.delete(fileContextTestHelper.getTestRootPath(fc), true); } private void cleanupFile(FileContext fc, Path name) throws IOException { @@ -93,8 +102,8 @@ public abstract class FileContextPermissionBase { return; } String filename = "foo"; - Path f = getTestRootPath(fc, filename); - createFile(fc, filename); + Path f = fileContextTestHelper.getTestRootPath(fc, filename); + fileContextTestHelper.createFile(fc, filename); doFilePermissionCheck(FileContext.FILE_DEFAULT_PERM.applyUMask(fc.getUMask()), fc.getFileStatus(f).getPermission()); } @@ -108,7 +117,7 @@ public abstract class FileContextPermissionBase { } String filename = "foo"; - Path f = getTestRootPath(fc, filename); + Path f = fileContextTestHelper.getTestRootPath(fc, filename); createFile(fc, f); try { @@ -133,7 +142,7 @@ public abstract class FileContextPermissionBase { } String filename = "bar"; - Path f = getTestRootPath(fc, filename); + Path f = fileContextTestHelper.getTestRootPath(fc, filename); createFile(fc, f); List groups = null; try { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java index da9b895b57e..fd7555690e4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java @@ -43,6 +43,7 @@ public abstract class FileContextSymlinkBaseTest { static final int blockSize = 8192; static final int fileSize = 16384; + protected final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); protected static FileContext fc; abstract protected String getScheme(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java index 6f0e23e55ba..d40528b3082 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.FileNotFoundException; import java.util.EnumSet; +import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.CreateOpts.BlockSize; import org.apache.hadoop.io.IOUtils; @@ -31,15 +32,18 @@ import org.junit.Assert; * Helper class for unit tests. */ public final class FileContextTestHelper { - // The test root is relative to the /build/test/data by default - public static final String TEST_ROOT_DIR = - System.getProperty("test.build.data", "build/test/data") + "/test"; private static final int DEFAULT_BLOCK_SIZE = 1024; private static final int DEFAULT_NUM_BLOCKS = 2; - private static String absTestRootDir = null; - /** Hidden constructor */ - private FileContextTestHelper() {} + private final String testRootDir; + private String absTestRootDir = null; + + /** + * Create a context with test root relative to the /build/test/data + */ + public FileContextTestHelper() { + testRootDir = System.getProperty("test.build.data", "target/test/data") + "/" + RandomStringUtils.randomAlphanumeric(10); + } public static int getDefaultBlockSize() { return DEFAULT_BLOCK_SIZE; @@ -53,36 +57,36 @@ public final class FileContextTestHelper { return data; } - public static Path getTestRootPath(FileContext fc) { - return fc.makeQualified(new Path(TEST_ROOT_DIR)); + public Path getTestRootPath(FileContext fc) { + return fc.makeQualified(new Path(testRootDir)); } - public static Path getTestRootPath(FileContext fc, String pathString) { - return fc.makeQualified(new Path(TEST_ROOT_DIR, pathString)); + public Path getTestRootPath(FileContext fc, String pathString) { + return fc.makeQualified(new Path(testRootDir, pathString)); } // the getAbsolutexxx method is needed because the root test dir // can be messed up by changing the working dir. - public static String getAbsoluteTestRootDir(FileContext fc) + public String getAbsoluteTestRootDir(FileContext fc) throws IOException { if (absTestRootDir == null) { - if (new Path(TEST_ROOT_DIR).isAbsolute()) { - absTestRootDir = TEST_ROOT_DIR; + if (new Path(testRootDir).isAbsolute()) { + absTestRootDir = testRootDir; } else { absTestRootDir = fc.getWorkingDirectory().toString() + "/" - + TEST_ROOT_DIR; + + testRootDir; } } return absTestRootDir; } - public static Path getAbsoluteTestRootPath(FileContext fc) throws IOException { + public Path getAbsoluteTestRootPath(FileContext fc) throws IOException { return fc.makeQualified(new Path(getAbsoluteTestRootDir(fc))); } - public static Path getDefaultWorkingDirectory(FileContext fc) + public Path getDefaultWorkingDirectory(FileContext fc) throws IOException { return getTestRootPath(fc, "/user/" + System.getProperty("user.name")) .makeQualified(fc.getDefaultFileSystem().getUri(), @@ -116,12 +120,12 @@ public final class FileContextTestHelper { return createFile(fc, path, DEFAULT_NUM_BLOCKS, CreateOpts.createParent()); } - public static long createFile(FileContext fc, String name) throws IOException { + public long createFile(FileContext fc, String name) throws IOException { Path path = getTestRootPath(fc, name); return createFile(fc, path); } - public static long createFileNonRecursive(FileContext fc, String name) + public long createFileNonRecursive(FileContext fc, String name) throws IOException { Path path = getTestRootPath(fc, name); return createFileNonRecursive(fc, path); @@ -190,7 +194,7 @@ public final class FileContextTestHelper { return buffer; } - public static FileStatus containsPath(FileContext fc, Path path, + public FileStatus containsPath(FileContext fc, Path path, FileStatus[] dirList) throws IOException { return containsPath(getTestRootPath(fc, path.toString()), dirList); @@ -206,7 +210,7 @@ public final class FileContextTestHelper { return null; } - public static FileStatus containsPath(FileContext fc, String path, + public FileStatus containsPath(FileContext fc, String path, FileStatus[] dirList) throws IOException { return containsPath(fc, new Path(path), dirList); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java index 10899338519..c473bdfe7b8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.fs; -import static org.apache.hadoop.fs.FileContextTestHelper.getTestRootPath; import static org.apache.hadoop.fs.FileContextTestHelper.readFile; import static org.apache.hadoop.fs.FileContextTestHelper.writeFile; import static org.junit.Assert.assertTrue; @@ -44,6 +43,7 @@ import org.junit.Test; *

*/ public abstract class FileContextUtilBase { + protected final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); protected FileContext fc; { @@ -58,19 +58,19 @@ public abstract class FileContextUtilBase { @Before public void setUp() throws Exception { - fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DEFAULT_PERM, true); } @After public void tearDown() throws Exception { - fc.delete(getTestRootPath(fc), true); + fc.delete(fileContextTestHelper.getTestRootPath(fc), true); } @Test public void testFcCopy() throws Exception{ final String ts = "some random text"; - Path file1 = getTestRootPath(fc, "file1"); - Path file2 = getTestRootPath(fc, "file2"); + Path file1 = fileContextTestHelper.getTestRootPath(fc, "file1"); + Path file2 = fileContextTestHelper.getTestRootPath(fc, "file2"); writeFile(fc, file1, ts.getBytes()); assertTrue(fc.util().exists(file1)); @@ -87,8 +87,8 @@ public abstract class FileContextUtilBase { public void testRecursiveFcCopy() throws Exception { final String ts = "some random text"; - Path dir1 = getTestRootPath(fc, "dir1"); - Path dir2 = getTestRootPath(fc, "dir2"); + Path dir1 = fileContextTestHelper.getTestRootPath(fc, "dir1"); + Path dir2 = fileContextTestHelper.getTestRootPath(fc, "dir2"); Path file1 = new Path(dir1, "file1"); fc.mkdir(dir1, null, false); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java index 213d31e9fee..a5d8403c66a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java @@ -23,27 +23,39 @@ import java.net.URI; import java.util.Random; +import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.token.Token; import org.junit.Assert; + import static org.junit.Assert.*; import static org.mockito.Mockito.mock; /** * Helper class for unit tests. */ -public final class FileSystemTestHelper { - // The test root is relative to the /build/test/data by default - public static final String TEST_ROOT_DIR = - System.getProperty("test.build.data", "target/test/data") + "/test"; +public class FileSystemTestHelper { private static final int DEFAULT_BLOCK_SIZE = 1024; private static final int DEFAULT_NUM_BLOCKS = 2; private static final short DEFAULT_NUM_REPL = 1; - private static String absTestRootDir = null; - /** Hidden constructor */ - private FileSystemTestHelper() {} - + protected final String testRootDir; + private String absTestRootDir = null; + + /** + * Create helper with test root located at /build/test/data + */ + public FileSystemTestHelper() { + this(System.getProperty("test.build.data", "target/test/data") + "/" + RandomStringUtils.randomAlphanumeric(10)); + } + + /** + * Create helper with the specified test root dir + */ + public FileSystemTestHelper(String testRootDir) { + this.testRootDir = testRootDir; + } + public static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException { FileSystem.addFileSystemForTesting(uri, conf, fs); @@ -61,19 +73,22 @@ public final class FileSystemTestHelper { return data; } + public String getTestRootDir() { + return testRootDir; + } /* * get testRootPath qualified for fSys */ - public static Path getTestRootPath(FileSystem fSys) { - return fSys.makeQualified(new Path(TEST_ROOT_DIR)); + public Path getTestRootPath(FileSystem fSys) { + return fSys.makeQualified(new Path(testRootDir)); } /* * get testRootPath + pathString qualified for fSys */ - public static Path getTestRootPath(FileSystem fSys, String pathString) { - return fSys.makeQualified(new Path(TEST_ROOT_DIR, pathString)); + public Path getTestRootPath(FileSystem fSys, String pathString) { + return fSys.makeQualified(new Path(testRootDir, pathString)); } @@ -82,25 +97,25 @@ public final class FileSystemTestHelper { // is often relative to the working directory of process // running the unit tests. - static String getAbsoluteTestRootDir(FileSystem fSys) + String getAbsoluteTestRootDir(FileSystem fSys) throws IOException { // NOTE: can't cache because of different filesystems! //if (absTestRootDir == null) - if (new Path(TEST_ROOT_DIR).isAbsolute()) { - absTestRootDir = TEST_ROOT_DIR; + if (new Path(testRootDir).isAbsolute()) { + absTestRootDir = testRootDir; } else { absTestRootDir = fSys.getWorkingDirectory().toString() + "/" - + TEST_ROOT_DIR; + + testRootDir; } //} return absTestRootDir; } - public static Path getAbsoluteTestRootPath(FileSystem fSys) throws IOException { + public Path getAbsoluteTestRootPath(FileSystem fSys) throws IOException { return fSys.makeQualified(new Path(getAbsoluteTestRootDir(fSys))); } - public static Path getDefaultWorkingDirectory(FileSystem fSys) + public Path getDefaultWorkingDirectory(FileSystem fSys) throws IOException { return getTestRootPath(fSys, "/user/" + System.getProperty("user.name")) .makeQualified(fSys.getUri(), @@ -136,7 +151,7 @@ public final class FileSystemTestHelper { return createFile(fSys, path, DEFAULT_NUM_BLOCKS, DEFAULT_BLOCK_SIZE, DEFAULT_NUM_REPL, true); } - public static long createFile(FileSystem fSys, String name) throws IOException { + public long createFile(FileSystem fSys, String name) throws IOException { Path path = getTestRootPath(fSys, name); return createFile(fSys, path); } @@ -188,7 +203,7 @@ public final class FileSystemTestHelper { return s; } - public static FileStatus containsPath(FileSystem fSys, Path path, + public FileStatus containsPath(FileSystem fSys, Path path, FileStatus[] dirList) throws IOException { for(int i = 0; i < dirList.length; i ++) { @@ -210,7 +225,7 @@ public final class FileSystemTestHelper { } - public static FileStatus containsPath(FileSystem fSys, String path, FileStatus[] dirList) + public FileStatus containsPath(FileSystem fSys, String path, FileStatus[] dirList) throws IOException { return containsPath(fSys, new Path(path), dirList); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java index e0ee5f03f27..e3932da05c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFSMainOperationsLocalFileSystem.java @@ -28,10 +28,14 @@ import org.junit.Test; public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTest { + @Override + protected FileSystem createFileSystem() throws IOException { + return FileSystem.getLocal(new Configuration()); + } + @Override @Before public void setUp() throws Exception { - fSys = FileSystem.getLocal(new Configuration()); super.setUp(); } @@ -52,8 +56,7 @@ public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTes @Test @Override public void testWDAbsolute() throws IOException { - Path absoluteDir = FileSystemTestHelper.getTestRootPath(fSys, - "test/existingDir"); + Path absoluteDir = getTestRootPath(fSys, "test/existingDir"); fSys.mkdirs(absoluteDir); fSys.setWorkingDirectory(absoluteDir); Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFcLocalFsPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFcLocalFsPermission.java index 439ce2c15ca..848eeb01961 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFcLocalFsPermission.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFcLocalFsPermission.java @@ -29,7 +29,6 @@ public class TestFcLocalFsPermission extends @Override @Before public void setUp() throws Exception { - fc = FileContext.getLocalFSFileContext(); super.setUp(); } @@ -39,4 +38,9 @@ public class TestFcLocalFsPermission extends super.tearDown(); } + @Override + protected FileContext getFileContext() throws UnsupportedFileSystemException { + return FileContext.getLocalFSFileContext(); + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextDeleteOnExit.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextDeleteOnExit.java index 2d536d30c0a..2a058f76249 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextDeleteOnExit.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextDeleteOnExit.java @@ -35,6 +35,7 @@ public class TestFileContextDeleteOnExit { private static int blockSize = 1024; private static int numBlocks = 2; + private final FileContextTestHelper helper = new FileContextTestHelper(); private FileContext fc; @Before @@ -44,7 +45,7 @@ public class TestFileContextDeleteOnExit { @After public void tearDown() throws IOException { - fc.delete(getTestRootPath(fc), true); + fc.delete(helper.getTestRootPath(fc), true); } @@ -60,7 +61,7 @@ public class TestFileContextDeleteOnExit { @Test public void testDeleteOnExit() throws Exception { // Create deleteOnExit entries - Path file1 = getTestRootPath(fc, "file1"); + Path file1 = helper.getTestRootPath(fc, "file1"); createFile(fc, file1, numBlocks, blockSize); fc.deleteOnExit(file1); checkDeleteOnExitData(1, fc, file1); @@ -68,12 +69,12 @@ public class TestFileContextDeleteOnExit { // Ensure shutdown hook is added Assert.assertTrue(ShutdownHookManager.get().hasShutdownHook(FileContext.FINALIZER)); - Path file2 = getTestRootPath(fc, "dir1/file2"); + Path file2 = helper.getTestRootPath(fc, "dir1/file2"); createFile(fc, file2, numBlocks, blockSize); fc.deleteOnExit(file2); checkDeleteOnExitData(1, fc, file1, file2); - Path dir = getTestRootPath(fc, "dir3/dir4/dir5/dir6"); + Path dir = helper.getTestRootPath(fc, "dir3/dir4/dir5/dir6"); createFile(fc, dir, numBlocks, blockSize); fc.deleteOnExit(dir); checkDeleteOnExitData(1, fc, file1, file2, dir); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java index 6f9248902eb..e3e34c5082f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java @@ -52,7 +52,7 @@ public class TestLocalFSFileContextMainOperations extends FileContextMainOperati @Test public void testDefaultFilePermission() throws IOException { - Path file = FileContextTestHelper.getTestRootPath(fc, + Path file = fileContextTestHelper.getTestRootPath(fc, "testDefaultFilePermission"); FileContextTestHelper.createFile(fc, file); FsPermission expect = FileContext.FILE_DEFAULT_PERM.applyUMask(fc.getUMask()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextSymlink.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextSymlink.java index 714683b5cc7..14924313e5d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextSymlink.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextSymlink.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileUtil; -import static org.apache.hadoop.fs.FileContextTestHelper.*; import static org.junit.Assert.*; import static org.junit.Assume.assumeTrue; import org.junit.Test; @@ -43,12 +42,12 @@ public class TestLocalFSFileContextSymlink extends FileContextSymlinkBaseTest { @Override protected String testBaseDir1() throws IOException { - return getAbsoluteTestRootDir(fc)+"/test1"; + return fileContextTestHelper.getAbsoluteTestRootDir(fc)+"/test1"; } @Override protected String testBaseDir2() throws IOException { - return getAbsoluteTestRootDir(fc)+"/test2"; + return fileContextTestHelper.getAbsoluteTestRootDir(fc)+"/test2"; } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index bcc46008f8d..cebb1ee4ee0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -69,7 +69,7 @@ public class TestLocalFileSystem { /** * Test the capability of setting the working directory. */ - @Test(timeout = 1000) + @Test(timeout = 10000) public void testWorkingDirectory() throws IOException { Path origDir = fileSys.getWorkingDirectory(); Path subdir = new Path(TEST_ROOT_DIR, "new"); @@ -156,7 +156,7 @@ public class TestLocalFileSystem { } } - @Test(timeout = 1000) + @Test(timeout = 10000) public void testCopy() throws IOException { Path src = new Path(TEST_ROOT_DIR, "dingo"); Path dst = new Path(TEST_ROOT_DIR, "yak"); @@ -271,7 +271,7 @@ public class TestLocalFileSystem { stats[0].getPath().toUri().getPath()); } - @Test(timeout = 1000) + @Test(timeout = 10000) public void testReportChecksumFailure() throws IOException { base.mkdirs(); assertTrue(base.exists() && base.isDirectory()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java index 45e9bfb79c7..8a4552299bc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFsFCStatistics.java @@ -25,8 +25,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; -import static org.apache.hadoop.fs.FileContextTestHelper.*; - /** *

* Tests the File Context Statistics for {@link LocalFileSystem} @@ -39,12 +37,12 @@ public class TestLocalFsFCStatistics extends FCStatisticsBaseTest { @Before public void setUp() throws Exception { fc = FileContext.getLocalFSFileContext(); - fc.mkdir(getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true); } @After public void tearDown() throws Exception { - fc.delete(getTestRootPath(fc, "test"), true); + fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true); } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java index bc30c5a8f08..9bc399e88ce 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java @@ -97,37 +97,37 @@ public class TestCopyPreserveFlag { assertEquals(0, cmd.run(args)); } - @Test(timeout = 1000) + @Test(timeout = 10000) public void testPutWithP() throws Exception { run(new Put(), "-p", FROM.toString(), TO.toString()); assertAttributesPreserved(); } - @Test(timeout = 1000) + @Test(timeout = 10000) public void testPutWithoutP() throws Exception { run(new Put(), FROM.toString(), TO.toString()); assertAttributesChanged(); } - @Test(timeout = 1000) + @Test(timeout = 10000) public void testGetWithP() throws Exception { run(new Get(), "-p", FROM.toString(), TO.toString()); assertAttributesPreserved(); } - @Test(timeout = 1000) + @Test(timeout = 10000) public void testGetWithoutP() throws Exception { run(new Get(), FROM.toString(), TO.toString()); assertAttributesChanged(); } - @Test(timeout = 1000) + @Test(timeout = 10000) public void testCpWithP() throws Exception { run(new Cp(), "-p", FROM.toString(), TO.toString()); assertAttributesPreserved(); } - @Test(timeout = 1000) + @Test(timeout = 10000) public void testCpWithoutP() throws Exception { run(new Cp(), FROM.toString(), TO.toString()); assertAttributesChanged(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java index f2aace9472d..8454025ba3d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java @@ -40,13 +40,15 @@ public class TestChRootedFileSystem { FileSystem fSys; // The ChRoootedFs FileSystem fSysTarget; // Path chrootedTo; - + FileSystemTestHelper fileSystemTestHelper; + @Before public void setUp() throws Exception { // create the test root on local_fs Configuration conf = new Configuration(); fSysTarget = FileSystem.getLocal(conf); - chrootedTo = FileSystemTestHelper.getAbsoluteTestRootPath(fSysTarget); + fileSystemTestHelper = new FileSystemTestHelper(); + chrootedTo = fileSystemTestHelper.getAbsoluteTestRootPath(fSysTarget); // In case previous test was killed before cleanup fSysTarget.delete(chrootedTo, true); @@ -107,12 +109,12 @@ public class TestChRootedFileSystem { // Create file - FileSystemTestHelper.createFile(fSys, "/foo"); + fileSystemTestHelper.createFile(fSys, "/foo"); Assert.assertTrue(fSys.isFile(new Path("/foo"))); Assert.assertTrue(fSysTarget.isFile(new Path(chrootedTo, "foo"))); // Create file with recursive dir - FileSystemTestHelper.createFile(fSys, "/newDir/foo"); + fileSystemTestHelper.createFile(fSys, "/newDir/foo"); Assert.assertTrue(fSys.isFile(new Path("/newDir/foo"))); Assert.assertTrue(fSysTarget.isFile(new Path(chrootedTo,"newDir/foo"))); @@ -122,7 +124,7 @@ public class TestChRootedFileSystem { Assert.assertFalse(fSysTarget.exists(new Path(chrootedTo, "newDir/foo"))); // Create file with a 2 component dirs recursively - FileSystemTestHelper.createFile(fSys, "/newDir/newDir2/foo"); + fileSystemTestHelper.createFile(fSys, "/newDir/newDir2/foo"); Assert.assertTrue(fSys.isFile(new Path("/newDir/newDir2/foo"))); Assert.assertTrue(fSysTarget.isFile(new Path(chrootedTo,"newDir/newDir2/foo"))); @@ -135,11 +137,11 @@ public class TestChRootedFileSystem { @Test public void testMkdirDelete() throws IOException { - fSys.mkdirs(FileSystemTestHelper.getTestRootPath(fSys, "/dirX")); + fSys.mkdirs(fileSystemTestHelper.getTestRootPath(fSys, "/dirX")); Assert.assertTrue(fSys.isDirectory(new Path("/dirX"))); Assert.assertTrue(fSysTarget.isDirectory(new Path(chrootedTo,"dirX"))); - fSys.mkdirs(FileSystemTestHelper.getTestRootPath(fSys, "/dirX/dirY")); + fSys.mkdirs(fileSystemTestHelper.getTestRootPath(fSys, "/dirX/dirY")); Assert.assertTrue(fSys.isDirectory(new Path("/dirX/dirY"))); Assert.assertTrue(fSysTarget.isDirectory(new Path(chrootedTo,"dirX/dirY"))); @@ -157,11 +159,11 @@ public class TestChRootedFileSystem { @Test public void testRename() throws IOException { // Rename a file - FileSystemTestHelper.createFile(fSys, "/newDir/foo"); + fileSystemTestHelper.createFile(fSys, "/newDir/foo"); fSys.rename(new Path("/newDir/foo"), new Path("/newDir/fooBar")); Assert.assertFalse(fSys.exists(new Path("/newDir/foo"))); Assert.assertFalse(fSysTarget.exists(new Path(chrootedTo,"newDir/foo"))); - Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/newDir/fooBar"))); + Assert.assertTrue(fSys.isFile(fileSystemTestHelper.getTestRootPath(fSys,"/newDir/fooBar"))); Assert.assertTrue(fSysTarget.isFile(new Path(chrootedTo,"newDir/fooBar"))); @@ -170,7 +172,7 @@ public class TestChRootedFileSystem { fSys.rename(new Path("/newDir/dirFoo"), new Path("/newDir/dirFooBar")); Assert.assertFalse(fSys.exists(new Path("/newDir/dirFoo"))); Assert.assertFalse(fSysTarget.exists(new Path(chrootedTo,"newDir/dirFoo"))); - Assert.assertTrue(fSys.isDirectory(FileSystemTestHelper.getTestRootPath(fSys,"/newDir/dirFooBar"))); + Assert.assertTrue(fSys.isDirectory(fileSystemTestHelper.getTestRootPath(fSys,"/newDir/dirFooBar"))); Assert.assertTrue(fSysTarget.isDirectory(new Path(chrootedTo,"newDir/dirFooBar"))); } @@ -214,10 +216,10 @@ public class TestChRootedFileSystem { - FileSystemTestHelper.createFile(fSys, "/foo"); - FileSystemTestHelper.createFile(fSys, "/bar"); + fileSystemTestHelper.createFile(fSys, "/foo"); + fileSystemTestHelper.createFile(fSys, "/bar"); fSys.mkdirs(new Path("/dirX")); - fSys.mkdirs(FileSystemTestHelper.getTestRootPath(fSys, "/dirY")); + fSys.mkdirs(fileSystemTestHelper.getTestRootPath(fSys, "/dirY")); fSys.mkdirs(new Path("/dirX/dirXX")); dirPaths = fSys.listStatus(new Path("/")); @@ -307,7 +309,7 @@ public class TestChRootedFileSystem { @Test public void testResolvePath() throws IOException { Assert.assertEquals(chrootedTo, fSys.resolvePath(new Path("/"))); - FileSystemTestHelper.createFile(fSys, "/foo"); + fileSystemTestHelper.createFile(fSys, "/foo"); Assert.assertEquals(new Path(chrootedTo, "foo"), fSys.resolvePath(new Path("/foo"))); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java index c52280154d6..6078bc0caa2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java @@ -38,6 +38,7 @@ import org.junit.Before; import org.junit.Test; public class TestChRootedFs { + FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); FileContext fc; // The ChRoootedFs FileContext fcTarget; // Path chrootedTo; @@ -46,7 +47,7 @@ public class TestChRootedFs { public void setUp() throws Exception { // create the test root on local_fs fcTarget = FileContext.getLocalFSFileContext(); - chrootedTo = FileContextTestHelper.getAbsoluteTestRootPath(fcTarget); + chrootedTo = fileContextTestHelper.getAbsoluteTestRootPath(fcTarget); // In case previous test was killed before cleanup fcTarget.delete(chrootedTo, true); @@ -105,12 +106,12 @@ public class TestChRootedFs { // Create file - FileContextTestHelper.createFileNonRecursive(fc, "/foo"); + fileContextTestHelper.createFileNonRecursive(fc, "/foo"); Assert.assertTrue(isFile(fc, new Path("/foo"))); Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo, "foo"))); // Create file with recursive dir - FileContextTestHelper.createFile(fc, "/newDir/foo"); + fileContextTestHelper.createFile(fc, "/newDir/foo"); Assert.assertTrue(isFile(fc, new Path("/newDir/foo"))); Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo,"newDir/foo"))); @@ -120,7 +121,7 @@ public class TestChRootedFs { Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"newDir/foo"))); // Create file with a 2 component dirs recursively - FileContextTestHelper.createFile(fc, "/newDir/newDir2/foo"); + fileContextTestHelper.createFile(fc, "/newDir/newDir2/foo"); Assert.assertTrue(isFile(fc, new Path("/newDir/newDir2/foo"))); Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo,"newDir/newDir2/foo"))); @@ -133,11 +134,11 @@ public class TestChRootedFs { @Test public void testMkdirDelete() throws IOException { - fc.mkdir(FileContextTestHelper.getTestRootPath(fc, "/dirX"), FileContext.DEFAULT_PERM, false); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "/dirX"), FileContext.DEFAULT_PERM, false); Assert.assertTrue(isDir(fc, new Path("/dirX"))); Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"dirX"))); - fc.mkdir(FileContextTestHelper.getTestRootPath(fc, "/dirX/dirY"), FileContext.DEFAULT_PERM, false); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "/dirX/dirY"), FileContext.DEFAULT_PERM, false); Assert.assertTrue(isDir(fc, new Path("/dirX/dirY"))); Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"dirX/dirY"))); @@ -155,11 +156,11 @@ public class TestChRootedFs { @Test public void testRename() throws IOException { // Rename a file - FileContextTestHelper.createFile(fc, "/newDir/foo"); + fileContextTestHelper.createFile(fc, "/newDir/foo"); fc.rename(new Path("/newDir/foo"), new Path("/newDir/fooBar")); Assert.assertFalse(exists(fc, new Path("/newDir/foo"))); Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"newDir/foo"))); - Assert.assertTrue(isFile(fc, FileContextTestHelper.getTestRootPath(fc,"/newDir/fooBar"))); + Assert.assertTrue(isFile(fc, fileContextTestHelper.getTestRootPath(fc,"/newDir/fooBar"))); Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo,"newDir/fooBar"))); @@ -168,7 +169,7 @@ public class TestChRootedFs { fc.rename(new Path("/newDir/dirFoo"), new Path("/newDir/dirFooBar")); Assert.assertFalse(exists(fc, new Path("/newDir/dirFoo"))); Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"newDir/dirFoo"))); - Assert.assertTrue(isDir(fc, FileContextTestHelper.getTestRootPath(fc,"/newDir/dirFooBar"))); + Assert.assertTrue(isDir(fc, fileContextTestHelper.getTestRootPath(fc,"/newDir/dirFooBar"))); Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"newDir/dirFooBar"))); } @@ -202,10 +203,10 @@ public class TestChRootedFs { - FileContextTestHelper.createFileNonRecursive(fc, "/foo"); - FileContextTestHelper.createFileNonRecursive(fc, "/bar"); + fileContextTestHelper.createFileNonRecursive(fc, "/foo"); + fileContextTestHelper.createFileNonRecursive(fc, "/bar"); fc.mkdir(new Path("/dirX"), FileContext.DEFAULT_PERM, false); - fc.mkdir(FileContextTestHelper.getTestRootPath(fc, "/dirY"), + fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "/dirY"), FileContext.DEFAULT_PERM, false); fc.mkdir(new Path("/dirX/dirXX"), FileContext.DEFAULT_PERM, false); @@ -213,16 +214,16 @@ public class TestChRootedFs { Assert.assertEquals(4, dirPaths.length); // Note the the file status paths are the full paths on target - fs = FileContextTestHelper.containsPath(fcTarget, "foo", dirPaths); + fs = fileContextTestHelper.containsPath(fcTarget, "foo", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue(fs.isFile()); - fs = FileContextTestHelper.containsPath(fcTarget, "bar", dirPaths); + fs = fileContextTestHelper.containsPath(fcTarget, "bar", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue(fs.isFile()); - fs = FileContextTestHelper.containsPath(fcTarget, "dirX", dirPaths); + fs = fileContextTestHelper.containsPath(fcTarget, "dirX", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue(fs.isDirectory()); - fs = FileContextTestHelper.containsPath(fcTarget, "dirY", dirPaths); + fs = fileContextTestHelper.containsPath(fcTarget, "dirY", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue(fs.isDirectory()); } @@ -273,7 +274,7 @@ public class TestChRootedFs { fc.mkdir(new Path("newDir"), FileContext.DEFAULT_PERM, true); Assert.assertTrue(isDir(fc, new Path(absoluteDir, "newDir"))); - absoluteDir = getTestRootPath(fc, "nonexistingPath"); + absoluteDir = fileContextTestHelper.getTestRootPath(fc, "nonexistingPath"); try { fc.setWorkingDirectory(absoluteDir); Assert.fail("cd to non existing dir should have failed"); @@ -297,7 +298,7 @@ public class TestChRootedFs { @Test public void testResolvePath() throws IOException { Assert.assertEquals(chrootedTo, fc.getDefaultFileSystem().resolvePath(new Path("/"))); - FileContextTestHelper.createFile(fc, "/foo"); + fileContextTestHelper.createFile(fc, "/foo"); Assert.assertEquals(new Path(chrootedTo, "foo"), fc.getDefaultFileSystem().resolvePath(new Path("/foo"))); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java index de4b1e87ac5..6081f385706 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java @@ -23,7 +23,6 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSMainOperationsBaseTest; -import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.After; @@ -32,14 +31,19 @@ import org.junit.Before; import org.junit.Test; public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTest { - static FileSystem fcTarget; + FileSystem fcTarget; + + @Override + protected FileSystem createFileSystem() throws Exception { + return ViewFileSystemTestSetup.setupForViewFileSystem( + ViewFileSystemTestSetup.createConfig(), this, fcTarget); + } + @Override @Before public void setUp() throws Exception { Configuration conf = new Configuration(); fcTarget = FileSystem.getLocal(conf); - fSys = ViewFileSystemTestSetup.setupForViewFileSystem( - ViewFileSystemTestSetup.createConfig(), fcTarget); super.setUp(); } @@ -47,14 +51,13 @@ public class TestFSMainOperationsLocalFileSystem extends FSMainOperationsBaseTes @After public void tearDown() throws Exception { super.tearDown(); - ViewFileSystemTestSetup.tearDown(fcTarget); + ViewFileSystemTestSetup.tearDown(this, fcTarget); } @Test @Override public void testWDAbsolute() throws IOException { - Path absoluteDir = FileSystemTestHelper.getTestRootPath(fSys, - "test/existingDir"); + Path absoluteDir = getTestRootPath(fSys, "test/existingDir"); fSys.mkdirs(absoluteDir); fSys.setWorkingDirectory(absoluteDir); Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcCreateMkdirLocalFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcCreateMkdirLocalFs.java index 16b38b72ec1..79217b3f421 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcCreateMkdirLocalFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcCreateMkdirLocalFs.java @@ -31,7 +31,7 @@ public class TestFcCreateMkdirLocalFs extends @Override @Before public void setUp() throws Exception { - fc = ViewFsTestSetup.setupForViewFsLocalFs(); + fc = ViewFsTestSetup.setupForViewFsLocalFs(fileContextTestHelper); super.setUp(); } @@ -39,6 +39,6 @@ public class TestFcCreateMkdirLocalFs extends @After public void tearDown() throws Exception { super.tearDown(); - ViewFsTestSetup.tearDownForViewFsLocalFs(); + ViewFsTestSetup.tearDownForViewFsLocalFs(fileContextTestHelper); } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java index 74f558509f1..61e0322491a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcMainOperationsLocalFs.java @@ -39,7 +39,7 @@ public class TestFcMainOperationsLocalFs extends @Override @Before public void setUp() throws Exception { - fc = ViewFsTestSetup.setupForViewFsLocalFs(); + fc = ViewFsTestSetup.setupForViewFsLocalFs(fileContextTestHelper); super.setUp(); } @@ -47,6 +47,6 @@ public class TestFcMainOperationsLocalFs extends @After public void tearDown() throws Exception { super.tearDown(); - ViewFsTestSetup.tearDownForViewFsLocalFs(); + ViewFsTestSetup.tearDownForViewFsLocalFs(fileContextTestHelper); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcPermissionsLocalFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcPermissionsLocalFs.java index 0e44be9be85..afbbd635e9f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcPermissionsLocalFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFcPermissionsLocalFs.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.viewfs; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextPermissionBase; import org.junit.After; @@ -26,11 +27,9 @@ import org.junit.Before; public class TestFcPermissionsLocalFs extends FileContextPermissionBase { - @Override @Before public void setUp() throws Exception { - fc = ViewFsTestSetup.setupForViewFsLocalFs(); super.setUp(); } @@ -38,6 +37,11 @@ public class TestFcPermissionsLocalFs extends FileContextPermissionBase { @After public void tearDown() throws Exception { super.tearDown(); - ViewFsTestSetup.tearDownForViewFsLocalFs(); + ViewFsTestSetup.tearDownForViewFsLocalFs(fileContextTestHelper); + } + + @Override + protected FileContext getFileContext() throws Exception { + return ViewFsTestSetup.setupForViewFsLocalFs(fileContextTestHelper); } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java index e320f87bbce..487ce91a078 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java @@ -52,7 +52,7 @@ public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup { conf.set("fs."+scheme+".impl", clazz.getName()); FakeFileSystem fs = (FakeFileSystem)FileSystem.get(uri, conf); assertEquals(uri, fs.getUri()); - Path targetPath = FileSystemTestHelper.getAbsoluteTestRootPath(fs); + Path targetPath = new FileSystemTestHelper().getAbsoluteTestRootPath(fs); ConfigUtil.addLink(conf, "/mounts/"+scheme, targetPath.toUri()); return fs; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java index 4b45fc8c5b8..49437924ea3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java @@ -38,7 +38,6 @@ import org.junit.Before; public class TestViewFileSystemLocalFileSystem extends ViewFileSystemBaseTest { - @Override @Before public void setUp() throws Exception { @@ -51,7 +50,7 @@ public class TestViewFileSystemLocalFileSystem extends ViewFileSystemBaseTest { @Override @After public void tearDown() throws Exception { - fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true); + fsTarget.delete(fileSystemTestHelper.getTestRootPath(fsTarget), true); super.tearDown(); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java index 4786cd5fdf5..877c2228c1e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java @@ -59,7 +59,7 @@ public class TestViewFileSystemWithAuthorityLocalFileSystem extends ViewFileSyst @Override @After public void tearDown() throws Exception { - fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true); + fsTarget.delete(fileSystemTestHelper.getTestRootPath(fsTarget), true); super.tearDown(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java index 4325f403462..62ef9d14638 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java @@ -35,11 +35,12 @@ public class TestViewFsTrash { FileSystem fsTarget; // the target file system - the mount will point here FileSystem fsView; Configuration conf; + FileSystemTestHelper fileSystemTestHelper = new FileSystemTestHelper(); - static class TestLFS extends LocalFileSystem { + class TestLFS extends LocalFileSystem { Path home; TestLFS() throws IOException { - this(new Path(FileSystemTestHelper.TEST_ROOT_DIR)); + this(new Path(fileSystemTestHelper.getTestRootDir())); } TestLFS(Path home) throws IOException { super(); @@ -54,23 +55,23 @@ public class TestViewFsTrash { @Before public void setUp() throws Exception { fsTarget = FileSystem.getLocal(new Configuration()); - fsTarget.mkdirs(new Path(FileSystemTestHelper. + fsTarget.mkdirs(new Path(fileSystemTestHelper. getTestRootPath(fsTarget), "dir1")); conf = ViewFileSystemTestSetup.createConfig(); - fsView = ViewFileSystemTestSetup.setupForViewFileSystem(conf, fsTarget); + fsView = ViewFileSystemTestSetup.setupForViewFileSystem(conf, fileSystemTestHelper, fsTarget); conf.set("fs.defaultFS", FsConstants.VIEWFS_URI.toString()); } @After public void tearDown() throws Exception { - ViewFileSystemTestSetup.tearDown(fsTarget); + ViewFileSystemTestSetup.tearDown(fileSystemTestHelper, fsTarget); fsTarget.delete(new Path(fsTarget.getHomeDirectory(), ".Trash/Current"), true); } @Test public void testTrash() throws IOException { - TestTrash.trashShell(conf, FileSystemTestHelper.getTestRootPath(fsView), + TestTrash.trashShell(conf, fileSystemTestHelper.getTestRootPath(fsView), fsTarget, new Path(fsTarget.getHomeDirectory(), ".Trash/Current")); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index f59e3ab4526..e5fcef7e4de 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -68,7 +68,15 @@ public class ViewFileSystemBaseTest { FileSystem fsTarget; // the target file system - the mount will point here Path targetTestRoot; Configuration conf; + final FileSystemTestHelper fileSystemTestHelper; + public ViewFileSystemBaseTest() { + this.fileSystemTestHelper = createFileSystemHelper(); + } + + protected FileSystemTestHelper createFileSystemHelper() { + return new FileSystemTestHelper(); + } @Before public void setUp() throws Exception { @@ -94,11 +102,11 @@ public class ViewFileSystemBaseTest { @After public void tearDown() throws Exception { - fsTarget.delete(FileSystemTestHelper.getTestRootPath(fsTarget), true); + fsTarget.delete(fileSystemTestHelper.getTestRootPath(fsTarget), true); } void initializeTargetTestRoot() throws IOException { - targetTestRoot = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget); + targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget); // In case previous test was killed before cleanup fsTarget.delete(targetTestRoot, true); @@ -199,7 +207,7 @@ public class ViewFileSystemBaseTest { @Test public void testOperationsThroughMountLinks() throws IOException { // Create file - FileSystemTestHelper.createFile(fsView, "/user/foo"); + fileSystemTestHelper.createFile(fsView, "/user/foo"); Assert.assertTrue("Created file should be type file", fsView.isFile(new Path("/user/foo"))); Assert.assertTrue("Target of created file should be type file", @@ -214,7 +222,7 @@ public class ViewFileSystemBaseTest { fsTarget.exists(new Path(targetTestRoot,"user/foo"))); // Create file with a 2 component dirs - FileSystemTestHelper.createFile(fsView, "/internalDir/linkToDir2/foo"); + fileSystemTestHelper.createFile(fsView, "/internalDir/linkToDir2/foo"); Assert.assertTrue("Created file should be type file", fsView.isFile(new Path("/internalDir/linkToDir2/foo"))); Assert.assertTrue("Target of created file should be type file", @@ -230,14 +238,14 @@ public class ViewFileSystemBaseTest { // Create file with a 3 component dirs - FileSystemTestHelper.createFile(fsView, "/internalDir/internalDir2/linkToDir3/foo"); + fileSystemTestHelper.createFile(fsView, "/internalDir/internalDir2/linkToDir3/foo"); Assert.assertTrue("Created file should be type file", fsView.isFile(new Path("/internalDir/internalDir2/linkToDir3/foo"))); Assert.assertTrue("Target of created file should be type file", fsTarget.isFile(new Path(targetTestRoot,"dir3/foo"))); // Recursive Create file with missing dirs - FileSystemTestHelper.createFile(fsView, + fileSystemTestHelper.createFile(fsView, "/internalDir/linkToDir2/missingDir/miss2/foo"); Assert.assertTrue("Created file should be type file", fsView.isFile(new Path("/internalDir/linkToDir2/missingDir/miss2/foo"))); @@ -256,14 +264,14 @@ public class ViewFileSystemBaseTest { // mkdir - fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX")); + fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX")); Assert.assertTrue("New dir should be type dir", fsView.isDirectory(new Path("/user/dirX"))); Assert.assertTrue("Target of new dir should be of type dir", fsTarget.isDirectory(new Path(targetTestRoot,"user/dirX"))); fsView.mkdirs( - FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX/dirY")); + fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX/dirY")); Assert.assertTrue("New dir should be type dir", fsView.isDirectory(new Path("/user/dirX/dirY"))); Assert.assertTrue("Target of new dir should be of type dir", @@ -285,14 +293,14 @@ public class ViewFileSystemBaseTest { Assert.assertFalse(fsTarget.exists(new Path(targetTestRoot,"user/dirX"))); // Rename a file - FileSystemTestHelper.createFile(fsView, "/user/foo"); + fileSystemTestHelper.createFile(fsView, "/user/foo"); fsView.rename(new Path("/user/foo"), new Path("/user/fooBar")); Assert.assertFalse("Renamed src should not exist", fsView.exists(new Path("/user/foo"))); Assert.assertFalse("Renamed src should not exist in target", fsTarget.exists(new Path(targetTestRoot,"user/foo"))); Assert.assertTrue("Renamed dest should exist as file", - fsView.isFile(FileSystemTestHelper.getTestRootPath(fsView,"/user/fooBar"))); + fsView.isFile(fileSystemTestHelper.getTestRootPath(fsView,"/user/fooBar"))); Assert.assertTrue("Renamed dest should exist as file in target", fsTarget.isFile(new Path(targetTestRoot,"user/fooBar"))); @@ -303,7 +311,7 @@ public class ViewFileSystemBaseTest { Assert.assertFalse("Renamed src should not exist in target", fsTarget.exists(new Path(targetTestRoot,"user/dirFoo"))); Assert.assertTrue("Renamed dest should exist as dir", - fsView.isDirectory(FileSystemTestHelper.getTestRootPath(fsView,"/user/dirFooBar"))); + fsView.isDirectory(fileSystemTestHelper.getTestRootPath(fsView,"/user/dirFooBar"))); Assert.assertTrue("Renamed dest should exist as dir in target", fsTarget.isDirectory(new Path(targetTestRoot,"user/dirFooBar"))); @@ -322,7 +330,7 @@ public class ViewFileSystemBaseTest { // rename across mount points that point to same target also fail @Test(expected=IOException.class) public void testRenameAcrossMounts1() throws IOException { - FileSystemTestHelper.createFile(fsView, "/user/foo"); + fileSystemTestHelper.createFile(fsView, "/user/foo"); fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); /* - code if we had wanted this to suceed Assert.assertFalse(fSys.exists(new Path("/user/foo"))); @@ -338,7 +346,7 @@ public class ViewFileSystemBaseTest { @Test(expected=IOException.class) public void testRenameAcrossMounts2() throws IOException { - FileSystemTestHelper.createFile(fsView, "/user/foo"); + fileSystemTestHelper.createFile(fsView, "/user/foo"); fsView.rename(new Path("/user/foo"), new Path("/data/fooBar")); } @@ -394,19 +402,19 @@ public class ViewFileSystemBaseTest { FileStatus[] dirPaths = fsView.listStatus(new Path("/")); FileStatus fs; Assert.assertEquals(getExpectedDirPaths(), dirPaths.length); - fs = FileSystemTestHelper.containsPath(fsView, "/user", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/user", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); - fs = FileSystemTestHelper.containsPath(fsView, "/data", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/data", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); - fs = FileSystemTestHelper.containsPath(fsView, "/internalDir", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/internalDir", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isDirectory()); - fs = FileSystemTestHelper.containsPath(fsView, "/danglingLink", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/danglingLink", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); - fs = FileSystemTestHelper.containsPath(fsView, "/linkToAFile", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/linkToAFile", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); @@ -416,10 +424,10 @@ public class ViewFileSystemBaseTest { dirPaths = fsView.listStatus(new Path("/internalDir")); Assert.assertEquals(2, dirPaths.length); - fs = FileSystemTestHelper.containsPath(fsView, "/internalDir/internalDir2", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/internalDir/internalDir2", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isDirectory()); - fs = FileSystemTestHelper.containsPath(fsView, "/internalDir/linkToDir2", + fs = fileSystemTestHelper.containsPath(fsView, "/internalDir/linkToDir2", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); @@ -436,22 +444,22 @@ public class ViewFileSystemBaseTest { Assert.assertEquals(0, dirPaths.length); // add a file - long len = FileSystemTestHelper.createFile(fsView, "/data/foo"); + long len = fileSystemTestHelper.createFile(fsView, "/data/foo"); dirPaths = fsView.listStatus(new Path("/data")); Assert.assertEquals(1, dirPaths.length); - fs = FileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("Created file shoudl appear as a file", fs.isFile()); Assert.assertEquals(len, fs.getLen()); // add a dir - fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/data/dirX")); + fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/data/dirX")); dirPaths = fsView.listStatus(new Path("/data")); Assert.assertEquals(2, dirPaths.length); - fs = FileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("Created file shoudl appear as a file", fs.isFile()); - fs = FileSystemTestHelper.containsPath(fsView, "/data/dirX", dirPaths); + fs = fileSystemTestHelper.containsPath(fsView, "/data/dirX", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("Created dir should appear as a dir", fs.isDirectory()); } @@ -487,12 +495,12 @@ public class ViewFileSystemBaseTest { public void testResolvePathInternalPaths() throws IOException { Assert.assertEquals(new Path("/"), fsView.resolvePath(new Path("/"))); Assert.assertEquals(new Path("/internalDir"), - fsView.resolvePath(new Path("/internalDir"))); + fsView.resolvePath(new Path("/internalDir"))); } @Test public void testResolvePathMountPoints() throws IOException { Assert.assertEquals(new Path(targetTestRoot,"user"), - fsView.resolvePath(new Path("/user"))); + fsView.resolvePath(new Path("/user"))); Assert.assertEquals(new Path(targetTestRoot,"data"), fsView.resolvePath(new Path("/data"))); Assert.assertEquals(new Path(targetTestRoot,"dir2"), @@ -504,25 +512,25 @@ public class ViewFileSystemBaseTest { @Test public void testResolvePathThroughMountPoints() throws IOException { - FileSystemTestHelper.createFile(fsView, "/user/foo"); + fileSystemTestHelper.createFile(fsView, "/user/foo"); Assert.assertEquals(new Path(targetTestRoot,"user/foo"), - fsView.resolvePath(new Path("/user/foo"))); + fsView.resolvePath(new Path("/user/foo"))); fsView.mkdirs( - FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX")); + fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX")); Assert.assertEquals(new Path(targetTestRoot,"user/dirX"), fsView.resolvePath(new Path("/user/dirX"))); fsView.mkdirs( - FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX/dirY")); + fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX/dirY")); Assert.assertEquals(new Path(targetTestRoot,"user/dirX/dirY"), fsView.resolvePath(new Path("/user/dirX/dirY"))); } @Test(expected=FileNotFoundException.class) public void testResolvePathDanglingLink() throws IOException { - fsView.resolvePath(new Path("/danglingLink")); + fsView.resolvePath(new Path("/danglingLink")); } @Test(expected=FileNotFoundException.class) @@ -534,7 +542,7 @@ public class ViewFileSystemBaseTest { @Test(expected=FileNotFoundException.class) public void testResolvePathMissingThroughMountPoints2() throws IOException { fsView.mkdirs( - FileSystemTestHelper.getTestRootPath(fsView, "/user/dirX")); + fileSystemTestHelper.getTestRootPath(fsView, "/user/dirX")); fsView.resolvePath(new Path("/user/dirX/nonExisting")); } @@ -550,76 +558,76 @@ public class ViewFileSystemBaseTest { // Mkdir on existing internal mount table succeed except for / @Test(expected=AccessControlException.class) public void testInternalMkdirSlash() throws IOException { - fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/")); + fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/")); } public void testInternalMkdirExisting1() throws IOException { Assert.assertTrue("mkdir of existing dir should succeed", - fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, + fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/internalDir"))); } public void testInternalMkdirExisting2() throws IOException { Assert.assertTrue("mkdir of existing dir should succeed", - fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, + fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/internalDir/linkToDir2"))); } // Mkdir for new internal mount table should fail @Test(expected=AccessControlException.class) public void testInternalMkdirNew() throws IOException { - fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/dirNew")); + fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/dirNew")); } @Test(expected=AccessControlException.class) public void testInternalMkdirNew2() throws IOException { - fsView.mkdirs(FileSystemTestHelper.getTestRootPath(fsView, "/internalDir/dirNew")); + fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/internalDir/dirNew")); } // Create File on internal mount table should fail @Test(expected=AccessControlException.class) public void testInternalCreate1() throws IOException { - FileSystemTestHelper.createFile(fsView, "/foo"); // 1 component + fileSystemTestHelper.createFile(fsView, "/foo"); // 1 component } @Test(expected=AccessControlException.class) public void testInternalCreate2() throws IOException { // 2 component - FileSystemTestHelper.createFile(fsView, "/internalDir/foo"); + fileSystemTestHelper.createFile(fsView, "/internalDir/foo"); } @Test(expected=AccessControlException.class) public void testInternalCreateMissingDir() throws IOException { - FileSystemTestHelper.createFile(fsView, "/missingDir/foo"); + fileSystemTestHelper.createFile(fsView, "/missingDir/foo"); } @Test(expected=AccessControlException.class) public void testInternalCreateMissingDir2() throws IOException { - FileSystemTestHelper.createFile(fsView, "/missingDir/miss2/foo"); + fileSystemTestHelper.createFile(fsView, "/missingDir/miss2/foo"); } @Test(expected=AccessControlException.class) public void testInternalCreateMissingDir3() throws IOException { - FileSystemTestHelper.createFile(fsView, "/internalDir/miss2/foo"); + fileSystemTestHelper.createFile(fsView, "/internalDir/miss2/foo"); } // Delete on internal mount table should fail @Test(expected=FileNotFoundException.class) public void testInternalDeleteNonExisting() throws IOException { - fsView.delete(new Path("/NonExisting"), false); + fsView.delete(new Path("/NonExisting"), false); } @Test(expected=FileNotFoundException.class) public void testInternalDeleteNonExisting2() throws IOException { - fsView.delete(new Path("/internalDir/NonExisting"), false); + fsView.delete(new Path("/internalDir/NonExisting"), false); } @Test(expected=AccessControlException.class) public void testInternalDeleteExisting() throws IOException { - fsView.delete(new Path("/internalDir"), false); + fsView.delete(new Path("/internalDir"), false); } @Test(expected=AccessControlException.class) public void testInternalDeleteExisting2() throws IOException { - fsView.getFileStatus( + fsView.getFileStatus( new Path("/internalDir/linkToDir2")).isDirectory(); fsView.delete(new Path("/internalDir/linkToDir2"), false); } @@ -641,7 +649,7 @@ public class ViewFileSystemBaseTest { } @Test(expected=AccessControlException.class) public void testInternalRename2() throws IOException { - fsView.getFileStatus(new Path("/internalDir/linkToDir2")).isDirectory(); + fsView.getFileStatus(new Path("/internalDir/linkToDir2")).isDirectory(); fsView.rename(new Path("/internalDir/linkToDir2"), new Path("/internalDir/dir1")); } @@ -665,7 +673,7 @@ public class ViewFileSystemBaseTest { @Test public void testCreateNonRecursive() throws IOException { - Path path = FileSystemTestHelper.getTestRootPath(fsView, "/user/foo"); + Path path = fileSystemTestHelper.getTestRootPath(fsView, "/user/foo"); fsView.createNonRecursive(path, false, 1024, (short)1, 1024L, null); FileStatus status = fsView.getFileStatus(new Path("/user/foo")); Assert.assertTrue("Created file should be type file", diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java index 446b38e60b8..07bd0ce6e09 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java @@ -55,18 +55,18 @@ public class ViewFileSystemTestSetup { * @return return the ViewFS File context to be used for tests * @throws Exception */ - static public FileSystem setupForViewFileSystem(Configuration conf, FileSystem fsTarget) throws Exception { + static public FileSystem setupForViewFileSystem(Configuration conf, FileSystemTestHelper fileSystemTestHelper, FileSystem fsTarget) throws Exception { /** * create the test root on local_fs - the mount table will point here */ - Path targetOfTests = FileSystemTestHelper.getTestRootPath(fsTarget); + Path targetOfTests = fileSystemTestHelper.getTestRootPath(fsTarget); // In case previous test was killed before cleanup fsTarget.delete(targetOfTests, true); fsTarget.mkdirs(targetOfTests); // Set up viewfs link for test dir as described above - String testDir = FileSystemTestHelper.getTestRootPath(fsTarget).toUri() + String testDir = fileSystemTestHelper.getTestRootPath(fsTarget).toUri() .getPath(); linkUpFirstComponents(conf, testDir, fsTarget, "test dir"); @@ -91,14 +91,21 @@ public class ViewFileSystemTestSetup { * * delete the test directory in the target fs */ - static public void tearDown(FileSystem fsTarget) throws Exception { - Path targetOfTests = FileSystemTestHelper.getTestRootPath(fsTarget); + static public void tearDown(FileSystemTestHelper fileSystemTestHelper, FileSystem fsTarget) throws Exception { + Path targetOfTests = fileSystemTestHelper.getTestRootPath(fsTarget); fsTarget.delete(targetOfTests, true); } public static Configuration createConfig() { + return createConfig(true); + } + + public static Configuration createConfig(boolean disableCache) { Configuration conf = new Configuration(); conf.set("fs.viewfs.impl", ViewFileSystem.class.getName()); + if (disableCache) { + conf.set("fs.viewfs.impl.disable.cache", "true"); + } return conf; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 7f731de23e9..71894e6da97 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -75,6 +75,7 @@ public class ViewFsBaseTest { Configuration conf; FileContext xfcViewWithAuthority; // same as fsView but with authority URI schemeWithAuthority; + final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); @Before public void setUp() throws Exception { @@ -118,7 +119,7 @@ public class ViewFsBaseTest { } void initializeTargetTestRoot() throws IOException { - targetTestRoot = FileContextTestHelper.getAbsoluteTestRootPath(fcTarget); + targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fcTarget); // In case previous test was killed before cleanup fcTarget.delete(targetTestRoot, true); @@ -127,7 +128,7 @@ public class ViewFsBaseTest { @After public void tearDown() throws Exception { - fcTarget.delete(FileContextTestHelper.getTestRootPath(fcTarget), true); + fcTarget.delete(fileContextTestHelper.getTestRootPath(fcTarget), true); } @Test @@ -181,7 +182,7 @@ public class ViewFsBaseTest { @Test public void testOperationsThroughMountLinks() throws IOException { // Create file - FileContextTestHelper.createFileNonRecursive(fcView, "/user/foo"); + fileContextTestHelper.createFileNonRecursive(fcView, "/user/foo"); Assert.assertTrue("Create file should be file", isFile(fcView, new Path("/user/foo"))); Assert.assertTrue("Target of created file should be type file", @@ -196,7 +197,7 @@ public class ViewFsBaseTest { exists(fcTarget, new Path(targetTestRoot,"user/foo"))); // Create file with a 2 component dirs - FileContextTestHelper.createFileNonRecursive(fcView, + fileContextTestHelper.createFileNonRecursive(fcView, "/internalDir/linkToDir2/foo"); Assert.assertTrue("Created file should be type file", isFile(fcView, new Path("/internalDir/linkToDir2/foo"))); @@ -213,7 +214,7 @@ public class ViewFsBaseTest { // Create file with a 3 component dirs - FileContextTestHelper.createFileNonRecursive(fcView, + fileContextTestHelper.createFileNonRecursive(fcView, "/internalDir/internalDir2/linkToDir3/foo"); Assert.assertTrue("Created file should be of type file", isFile(fcView, new Path("/internalDir/internalDir2/linkToDir3/foo"))); @@ -221,7 +222,7 @@ public class ViewFsBaseTest { isFile(fcTarget, new Path(targetTestRoot,"dir3/foo"))); // Recursive Create file with missing dirs - FileContextTestHelper.createFile(fcView, + fileContextTestHelper.createFile(fcView, "/internalDir/linkToDir2/missingDir/miss2/foo"); Assert.assertTrue("Created file should be of type file", isFile(fcView, new Path("/internalDir/linkToDir2/missingDir/miss2/foo"))); @@ -239,14 +240,14 @@ public class ViewFsBaseTest { // mkdir - fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/user/dirX"), + fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/user/dirX"), FileContext.DEFAULT_PERM, false); Assert.assertTrue("New dir should be type dir", isDir(fcView, new Path("/user/dirX"))); Assert.assertTrue("Target of new dir should be of type dir", isDir(fcTarget, new Path(targetTestRoot,"user/dirX"))); - fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"), + fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"), FileContext.DEFAULT_PERM, false); Assert.assertTrue("New dir should be type dir", isDir(fcView, new Path("/user/dirX/dirY"))); @@ -270,13 +271,13 @@ public class ViewFsBaseTest { exists(fcTarget, new Path(targetTestRoot,"user/dirX"))); // Rename a file - FileContextTestHelper.createFile(fcView, "/user/foo"); + fileContextTestHelper.createFile(fcView, "/user/foo"); fcView.rename(new Path("/user/foo"), new Path("/user/fooBar")); Assert.assertFalse("Renamed src should not exist", exists(fcView, new Path("/user/foo"))); Assert.assertFalse(exists(fcTarget, new Path(targetTestRoot,"user/foo"))); Assert.assertTrue(isFile(fcView, - FileContextTestHelper.getTestRootPath(fcView,"/user/fooBar"))); + fileContextTestHelper.getTestRootPath(fcView,"/user/fooBar"))); Assert.assertTrue(isFile(fcTarget, new Path(targetTestRoot,"user/fooBar"))); fcView.mkdir(new Path("/user/dirFoo"), FileContext.DEFAULT_PERM, false); @@ -287,7 +288,7 @@ public class ViewFsBaseTest { exists(fcTarget, new Path(targetTestRoot,"user/dirFoo"))); Assert.assertTrue("Renamed dest should exist as dir", isDir(fcView, - FileContextTestHelper.getTestRootPath(fcView,"/user/dirFooBar"))); + fileContextTestHelper.getTestRootPath(fcView,"/user/dirFooBar"))); Assert.assertTrue("Renamed dest should exist as dir in target", isDir(fcTarget,new Path(targetTestRoot,"user/dirFooBar"))); @@ -309,7 +310,7 @@ public class ViewFsBaseTest { // rename across mount points that point to same target also fail @Test(expected=IOException.class) public void testRenameAcrossMounts1() throws IOException { - FileContextTestHelper.createFile(fcView, "/user/foo"); + fileContextTestHelper.createFile(fcView, "/user/foo"); fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar")); /* - code if we had wanted this to succeed Assert.assertFalse(exists(fc, new Path("/user/foo"))); @@ -327,7 +328,7 @@ public class ViewFsBaseTest { @Test(expected=IOException.class) public void testRenameAcrossMounts2() throws IOException { - FileContextTestHelper.createFile(fcView, "/user/foo"); + fileContextTestHelper.createFile(fcView, "/user/foo"); fcView.rename(new Path("/user/foo"), new Path("/data/fooBar")); } @@ -382,19 +383,19 @@ public class ViewFsBaseTest { FileStatus[] dirPaths = fcView.util().listStatus(new Path("/")); FileStatus fs; Assert.assertEquals(7, dirPaths.length); - fs = FileContextTestHelper.containsPath(fcView, "/user", dirPaths); + fs = fileContextTestHelper.containsPath(fcView, "/user", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); - fs = FileContextTestHelper.containsPath(fcView, "/data", dirPaths); + fs = fileContextTestHelper.containsPath(fcView, "/data", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); - fs = FileContextTestHelper.containsPath(fcView, "/internalDir", dirPaths); + fs = fileContextTestHelper.containsPath(fcView, "/internalDir", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("InternalDirs should appear as dir", fs.isDirectory()); - fs = FileContextTestHelper.containsPath(fcView, "/danglingLink", dirPaths); + fs = fileContextTestHelper.containsPath(fcView, "/danglingLink", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); - fs = FileContextTestHelper.containsPath(fcView, "/linkToAFile", dirPaths); + fs = fileContextTestHelper.containsPath(fcView, "/linkToAFile", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); @@ -404,11 +405,11 @@ public class ViewFsBaseTest { dirPaths = fcView.util().listStatus(new Path("/internalDir")); Assert.assertEquals(2, dirPaths.length); - fs = FileContextTestHelper.containsPath(fcView, + fs = fileContextTestHelper.containsPath(fcView, "/internalDir/internalDir2", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("InternalDirs should appear as dir",fs.isDirectory()); - fs = FileContextTestHelper.containsPath(fcView, + fs = fileContextTestHelper.containsPath(fcView, "/internalDir/linkToDir2", dirPaths); Assert.assertNotNull(fs); Assert.assertTrue("A mount should appear as symlink", fs.isSymlink()); @@ -534,19 +535,19 @@ public class ViewFsBaseTest { @Test public void testResolvePathThroughMountPoints() throws IOException { - FileContextTestHelper.createFile(fcView, "/user/foo"); + fileContextTestHelper.createFile(fcView, "/user/foo"); Assert.assertEquals(new Path(targetTestRoot,"user/foo"), fcView.resolvePath(new Path("/user/foo"))); fcView.mkdir( - FileContextTestHelper.getTestRootPath(fcView, "/user/dirX"), + fileContextTestHelper.getTestRootPath(fcView, "/user/dirX"), FileContext.DEFAULT_PERM, false); Assert.assertEquals(new Path(targetTestRoot,"user/dirX"), fcView.resolvePath(new Path("/user/dirX"))); fcView.mkdir( - FileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"), + fileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"), FileContext.DEFAULT_PERM, false); Assert.assertEquals(new Path(targetTestRoot,"user/dirX/dirY"), fcView.resolvePath(new Path("/user/dirX/dirY"))); @@ -566,7 +567,7 @@ public class ViewFsBaseTest { @Test(expected=FileNotFoundException.class) public void testResolvePathMissingThroughMountPoints2() throws IOException { fcView.mkdir( - FileContextTestHelper.getTestRootPath(fcView, "/user/dirX"), + fileContextTestHelper.getTestRootPath(fcView, "/user/dirX"), FileContext.DEFAULT_PERM, false); fcView.resolvePath(new Path("/user/dirX/nonExisting")); } @@ -584,29 +585,29 @@ public class ViewFsBaseTest { // Mkdir on internal mount table should fail @Test(expected=AccessControlException.class) public void testInternalMkdirSlash() throws IOException { - fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/"), + fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/"), FileContext.DEFAULT_PERM, false); } @Test(expected=AccessControlException.class) public void testInternalMkdirExisting1() throws IOException { - fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/internalDir"), + fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/internalDir"), FileContext.DEFAULT_PERM, false); } @Test(expected=AccessControlException.class) public void testInternalMkdirExisting2() throws IOException { - fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, + fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/internalDir/linkToDir2"), FileContext.DEFAULT_PERM, false); } @Test(expected=AccessControlException.class) public void testInternalMkdirNew() throws IOException { - fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/dirNew"), + fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/dirNew"), FileContext.DEFAULT_PERM, false); } @Test(expected=AccessControlException.class) public void testInternalMkdirNew2() throws IOException { - fcView.mkdir(FileContextTestHelper.getTestRootPath(fcView, "/internalDir/dirNew"), + fcView.mkdir(fileContextTestHelper.getTestRootPath(fcView, "/internalDir/dirNew"), FileContext.DEFAULT_PERM, false); } @@ -614,28 +615,28 @@ public class ViewFsBaseTest { @Test(expected=AccessControlException.class) public void testInternalCreate1() throws IOException { - FileContextTestHelper.createFileNonRecursive(fcView, "/foo"); // 1 component + fileContextTestHelper.createFileNonRecursive(fcView, "/foo"); // 1 component } @Test(expected=AccessControlException.class) public void testInternalCreate2() throws IOException { // 2 component - FileContextTestHelper.createFileNonRecursive(fcView, "/internalDir/foo"); + fileContextTestHelper.createFileNonRecursive(fcView, "/internalDir/foo"); } @Test(expected=AccessControlException.class) public void testInternalCreateMissingDir() throws IOException { - FileContextTestHelper.createFile(fcView, "/missingDir/foo"); + fileContextTestHelper.createFile(fcView, "/missingDir/foo"); } @Test(expected=AccessControlException.class) public void testInternalCreateMissingDir2() throws IOException { - FileContextTestHelper.createFile(fcView, "/missingDir/miss2/foo"); + fileContextTestHelper.createFile(fcView, "/missingDir/miss2/foo"); } @Test(expected=AccessControlException.class) public void testInternalCreateMissingDir3() throws IOException { - FileContextTestHelper.createFile(fcView, "/internalDir/miss2/foo"); + fileContextTestHelper.createFile(fcView, "/internalDir/miss2/foo"); } // Delete on internal mount table should fail diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java index ac63217fd43..0dd83408173 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsTestSetup.java @@ -53,12 +53,12 @@ public class ViewFsTestSetup { /* * return the ViewFS File context to be used for tests */ - static public FileContext setupForViewFsLocalFs() throws Exception { + static public FileContext setupForViewFsLocalFs(FileContextTestHelper helper) throws Exception { /** * create the test root on local_fs - the mount table will point here */ FileContext fsTarget = FileContext.getLocalFSFileContext(); - Path targetOfTests = FileContextTestHelper.getTestRootPath(fsTarget); + Path targetOfTests = helper.getTestRootPath(fsTarget); // In case previous test was killed before cleanup fsTarget.delete(targetOfTests, true); @@ -66,7 +66,7 @@ public class ViewFsTestSetup { Configuration conf = new Configuration(); // Set up viewfs link for test dir as described above - String testDir = FileContextTestHelper.getTestRootPath(fsTarget).toUri() + String testDir = helper.getTestRootPath(fsTarget).toUri() .getPath(); linkUpFirstComponents(conf, testDir, fsTarget, "test dir"); @@ -91,9 +91,9 @@ public class ViewFsTestSetup { * * delete the test directory in the target local fs */ - static public void tearDownForViewFsLocalFs() throws Exception { + static public void tearDownForViewFsLocalFs(FileContextTestHelper helper) throws Exception { FileContext fclocal = FileContext.getLocalFSFileContext(); - Path targetOfTests = FileContextTestHelper.getTestRootPath(fclocal); + Path targetOfTests = helper.getTestRootPath(fclocal); fclocal.delete(targetOfTests, true); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java index 9c68b282f63..89af8d06f1a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java @@ -23,7 +23,10 @@ import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.RandomAccessFile; import java.net.Socket; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -63,17 +66,13 @@ public abstract class ClientBaseWithFixes extends ZKTestCase { static final File BASETEST = new File(System.getProperty("build.test.dir", "build")); - protected String hostPort = "127.0.0.1:" + PortAssignment.unique(); + protected final String hostPort = initHostPort(); protected int maxCnxns = 0; protected ServerCnxnFactory serverFactory = null; protected File tmpDir = null; long initialFdCount; - public ClientBaseWithFixes() { - super(); - } - /** * In general don't use this. Only use in the special case that you * want to ignore results (for whatever reason) in your test. Don't @@ -154,6 +153,10 @@ public abstract class ClientBaseWithFixes extends ZKTestCase { private LinkedList allClients; private boolean allClientsSetup = false; + private RandomAccessFile portNumLockFile; + + private File portNumFile; + protected TestableZooKeeper createClient(CountdownWatcher watcher, String hp) throws IOException, InterruptedException { @@ -311,6 +314,7 @@ public abstract class ClientBaseWithFixes extends ZKTestCase { return tmpDir; } + private static int getPort(String hostPort) { String[] split = hostPort.split(":"); String portstr = split[split.length-1]; @@ -394,6 +398,35 @@ public abstract class ClientBaseWithFixes extends ZKTestCase { LOG.info("Client test setup finished"); } + private String initHostPort() { + BASETEST.mkdirs(); + int port; + for (;;) { + port = PortAssignment.unique(); + FileLock lock = null; + portNumLockFile = null; + try { + try { + portNumFile = new File(BASETEST, port + ".lock"); + portNumLockFile = new RandomAccessFile(portNumFile, "rw"); + try { + lock = portNumLockFile.getChannel().tryLock(); + } catch (OverlappingFileLockException e) { + continue; + } + } finally { + if (lock != null) + break; + if (portNumLockFile != null) + portNumLockFile.close(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return "127.0.0.1:" + port; + } + protected void startServer() throws Exception { LOG.info("STARTING server"); serverFactory = createNewServerInstance(tmpDir, serverFactory, hostPort, maxCnxns); @@ -434,6 +467,9 @@ public abstract class ClientBaseWithFixes extends ZKTestCase { stopServer(); + portNumLockFile.close(); + portNumFile.delete(); + if (tmpDir != null) { Assert.assertTrue("delete " + tmpDir.toString(), recursiveDelete(tmpDir)); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java index 1517c062b7d..7495c6ed66c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java @@ -39,8 +39,10 @@ import org.mockito.Mockito; public class TestSequenceFile extends TestCase { private static final Log LOG = LogFactory.getLog(TestSequenceFile.class); - private static Configuration conf = new Configuration(); + private Configuration conf = new Configuration(); + public TestSequenceFile() { } + public TestSequenceFile(String name) { super(name); } /** Unit tests for SequenceFile. */ @@ -126,7 +128,7 @@ public class TestSequenceFile extends TestCase { } } - private static void writeTest(FileSystem fs, int count, int seed, Path file, + private void writeTest(FileSystem fs, int count, int seed, Path file, CompressionType compressionType, CompressionCodec codec) throws IOException { fs.delete(file, true); @@ -146,7 +148,7 @@ public class TestSequenceFile extends TestCase { writer.close(); } - private static void readTest(FileSystem fs, int count, int seed, Path file) + private void readTest(FileSystem fs, int count, int seed, Path file) throws IOException { LOG.debug("reading " + count + " records"); SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); @@ -202,7 +204,7 @@ public class TestSequenceFile extends TestCase { } - private static void sortTest(FileSystem fs, int count, int megabytes, + private void sortTest(FileSystem fs, int count, int megabytes, int factor, boolean fast, Path file) throws IOException { fs.delete(new Path(file+".sorted"), true); @@ -212,7 +214,7 @@ public class TestSequenceFile extends TestCase { LOG.info("done sorting " + count + " debug"); } - private static void checkSort(FileSystem fs, int count, int seed, Path file) + private void checkSort(FileSystem fs, int count, int seed, Path file) throws IOException { LOG.info("sorting " + count + " records in memory for debug"); RandomDatum.Generator generator = new RandomDatum.Generator(seed); @@ -249,7 +251,7 @@ public class TestSequenceFile extends TestCase { LOG.debug("sucessfully checked " + count + " records"); } - private static void mergeTest(FileSystem fs, int count, int seed, Path file, + private void mergeTest(FileSystem fs, int count, int seed, Path file, CompressionType compressionType, boolean fast, int factor, int megabytes) throws IOException { @@ -293,7 +295,7 @@ public class TestSequenceFile extends TestCase { .merge(sortedNames, file.suffix(".sorted")); } - private static SequenceFile.Sorter newSorter(FileSystem fs, + private SequenceFile.Sorter newSorter(FileSystem fs, boolean fast, int megabytes, int factor) { SequenceFile.Sorter sorter = @@ -370,7 +372,7 @@ public class TestSequenceFile extends TestCase { } - private static SequenceFile.Metadata readMetadata(FileSystem fs, Path file) + private SequenceFile.Metadata readMetadata(FileSystem fs, Path file) throws IOException { LOG.info("reading file: " + file.toString()); SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); @@ -379,7 +381,7 @@ public class TestSequenceFile extends TestCase { return meta; } - private static void writeMetadataTest(FileSystem fs, int count, int seed, Path file, + private void writeMetadataTest(FileSystem fs, int count, int seed, Path file, CompressionType compressionType, CompressionCodec codec, SequenceFile.Metadata metadata) throws IOException { fs.delete(file, true); @@ -399,7 +401,7 @@ public class TestSequenceFile extends TestCase { writer.close(); } - private static void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata) + private void sortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile, SequenceFile.Metadata metadata) throws IOException { fs.delete(sortedFile, true); LOG.info("sorting: " + unsortedFile + " to: " + sortedFile); @@ -538,7 +540,6 @@ public class TestSequenceFile extends TestCase { } public void testRecursiveSeqFileCreate() throws IOException { - Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); Path name = new Path(new Path(System.getProperty("test.build.data","."), "recursiveCreateDir") , "file"); @@ -673,7 +674,9 @@ public class TestSequenceFile extends TestCase { } } - fs = file.getFileSystem(conf); + TestSequenceFile test = new TestSequenceFile(); + + fs = file.getFileSystem(test.conf); LOG.info("count = " + count); LOG.info("megabytes = " + megabytes); @@ -696,25 +699,25 @@ public class TestSequenceFile extends TestCase { CompressionType compressionType = CompressionType.valueOf(compressType); CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance( - conf.getClassByName(compressionCodec), - conf); + test.conf.getClassByName(compressionCodec), + test.conf); if (rwonly || (create && !merge)) { - writeTest(fs, count, seed, file, compressionType, codec); - readTest(fs, count, seed, file); + test.writeTest(fs, count, seed, file, compressionType, codec); + test.readTest(fs, count, seed, file); } if (!rwonly) { if (merge) { - mergeTest(fs, count, seed, file, compressionType, + test.mergeTest(fs, count, seed, file, compressionType, fast, factor, megabytes); } else { - sortTest(fs, count, megabytes, factor, fast, file); + test.sortTest(fs, count, megabytes, factor, fast, file); } } if (check) { - checkSort(fs, count, seed, file); + test.checkSort(fs, count, seed, file); } } finally { fs.close(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java index c9fc1eae4f5..a78c015d435 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileSerialization.java @@ -46,7 +46,7 @@ public class TestSequenceFileSerialization extends TestCase { public void testJavaSerialization() throws Exception { Path file = new Path(System.getProperty("test.build.data",".") + - "/test.seq"); + "/testseqser.seq"); fs.delete(file, true); Writer writer = SequenceFile.createWriter(fs, conf, file, Long.class, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java index 2682634516f..886cba14ec6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileByteArrays.java @@ -64,7 +64,7 @@ public class TestTFileByteArrays { private String compression = Compression.Algorithm.GZ.getName(); private String comparator = "memcmp"; - private String outputFile = "TFileTestByteArrays"; + private final String outputFile = getClass().getSimpleName(); /* * pre-sampled numbers of records in one block, based on the given the @@ -75,17 +75,16 @@ public class TestTFileByteArrays { private int records1stBlock = usingNative ? 5674 : 4480; private int records2ndBlock = usingNative ? 5574 : 4263; - public void init(String compression, String comparator, String outputFile, + public void init(String compression, String comparator, int numRecords1stBlock, int numRecords2ndBlock) { - init(compression, comparator, outputFile); + init(compression, comparator); this.records1stBlock = numRecords1stBlock; this.records2ndBlock = numRecords2ndBlock; } - public void init(String compression, String comparator, String outputFile) { + public void init(String compression, String comparator) { this.compression = compression; this.comparator = comparator; - this.outputFile = outputFile; } @Before @@ -99,7 +98,7 @@ public class TestTFileByteArrays { @After public void tearDown() throws IOException { if (!skip) - fs.delete(path, true); + fs.delete(path, true); } @Test diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileJClassComparatorByteArrays.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileJClassComparatorByteArrays.java index e33b3a544a5..ebb0fd163a9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileJClassComparatorByteArrays.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileJClassComparatorByteArrays.java @@ -37,8 +37,7 @@ public class TestTFileJClassComparatorByteArrays extends TestTFileByteArrays { @Override public void setUp() throws IOException { init(Compression.Algorithm.GZ.getName(), - "jclass: org.apache.hadoop.io.file.tfile.MyComparator", - "TFileTestJClassComparator"); + "jclass: org.apache.hadoop.io.file.tfile.MyComparator"); super.setUp(); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsByteArrays.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsByteArrays.java index c3f0e4e694c..1b3a351cd75 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsByteArrays.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsByteArrays.java @@ -34,8 +34,7 @@ public class TestTFileLzoCodecsByteArrays extends TestTFileByteArrays { } // TODO: sample the generated key/value records, and put the numbers below - init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo", - 2605, 2558); + init(Compression.Algorithm.LZO.getName(), "memcmp", 2605, 2558); if (!skip) super.setUp(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsStreams.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsStreams.java index 8ac911be9d0..9436a7aa004 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsStreams.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileLzoCodecsStreams.java @@ -32,7 +32,7 @@ public class TestTFileLzoCodecsStreams extends TestTFileStreams { if (skip) { System.out.println("Skipped"); } - init(Compression.Algorithm.LZO.getName(), "memcmp", "TFileTestCodecsLzo"); + init(Compression.Algorithm.LZO.getName(), "memcmp"); if (!skip) super.setUp(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsByteArrays.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsByteArrays.java index 6fde1d8c1dd..081c8273d77 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsByteArrays.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsByteArrays.java @@ -25,8 +25,7 @@ public class TestTFileNoneCodecsByteArrays extends TestTFileByteArrays { */ @Override public void setUp() throws IOException { - init(Compression.Algorithm.NONE.getName(), "memcmp", "TFileTestCodecsNone", - 24, 24); + init(Compression.Algorithm.NONE.getName(), "memcmp", 24, 24); super.setUp(); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsJClassComparatorByteArrays.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsJClassComparatorByteArrays.java index 7a2c2fc9c47..d7a579016d5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsJClassComparatorByteArrays.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsJClassComparatorByteArrays.java @@ -33,8 +33,7 @@ public class TestTFileNoneCodecsJClassComparatorByteArrays extends TestTFileByte @Override public void setUp() throws IOException { init(Compression.Algorithm.NONE.getName(), - "jclass: org.apache.hadoop.io.file.tfile.MyComparator", - "TestTFileNoneCodecsJClassComparatorByteArrays", 24, 24); + "jclass: org.apache.hadoop.io.file.tfile.MyComparator", 24, 24); super.setUp(); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsStreams.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsStreams.java index 54621f3c5b6..e06f51968e6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsStreams.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileNoneCodecsStreams.java @@ -26,7 +26,7 @@ public class TestTFileNoneCodecsStreams extends TestTFileStreams { */ @Override public void setUp() throws IOException { - init(Compression.Algorithm.NONE.getName(), "memcmp", "TFileTestCodecsNone"); + init(Compression.Algorithm.NONE.getName(), "memcmp"); super.setUp(); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java index 89ba0d1707c..e8da180700b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileStreams.java @@ -57,12 +57,11 @@ public class TestTFileStreams extends TestCase { private String compression = Compression.Algorithm.GZ.getName(); private String comparator = "memcmp"; - private String outputFile = "TFileTestStreams"; + private final String outputFile = getClass().getSimpleName(); - public void init(String compression, String comparator, String outputFile) { + public void init(String compression, String comparator) { this.compression = compression; this.comparator = comparator; - this.outputFile = outputFile; } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java index 7f53a7da41d..3b9861ea931 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java @@ -64,6 +64,9 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @RunWith(MockitoJUnitRunner.class) public class TestMetricsSystemImpl { private static final Log LOG = LogFactory.getLog(TestMetricsSystemImpl.class); + + static { DefaultMetricsSystem.setMiniClusterMode(true); } + @Captor private ArgumentCaptor r1; @Captor private ArgumentCaptor r2; private static String hostname = MetricsSystemImpl.getHostname(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsPermission.java index e2b684912b4..4852f3e6b2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsPermission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsPermission.java @@ -35,9 +35,22 @@ import org.junit.BeforeClass; public class TestFcHdfsPermission extends FileContextPermissionBase { + private static final FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); + private static FileContext fc; + private static MiniDFSCluster cluster; private static Path defaultWorkingDirectory; + @Override + protected FileContextTestHelper getFileContextHelper() { + return fileContextTestHelper; + } + + @Override + protected FileContext getFileContext() { + return fc; + } + @BeforeClass public static void clusterSetupAtBegining() throws IOException, LoginException, URISyntaxException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSetUMask.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSetUMask.java index 4da771b4401..52f032dd933 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSetUMask.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSetUMask.java @@ -42,6 +42,7 @@ import org.junit.Test; public class TestFcHdfsSetUMask { + private static FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); private static MiniDFSCluster cluster; private static Path defaultWorkingDirectory; private static FileContext fc; @@ -110,12 +111,12 @@ public class TestFcHdfsSetUMask { @Before public void setUp() throws Exception { fc.setUMask(WIDE_OPEN_TEST_UMASK); - fc.mkdir(getTestRootPath(fc), FileContext.DEFAULT_PERM, true); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc), FileContext.DEFAULT_PERM, true); } @After public void tearDown() throws Exception { - fc.delete(getTestRootPath(fc), true); + fc.delete(fileContextTestHelper.getTestRootPath(fc), true); } @Test @@ -199,7 +200,7 @@ public class TestFcHdfsSetUMask { public void testMkdirWithExistingDir(FsPermission umask, FsPermission expectedPerms) throws IOException { - Path f = getTestRootPath(fc, "aDir"); + Path f = fileContextTestHelper.getTestRootPath(fc, "aDir"); fc.setUMask(umask); fc.mkdir(f, FileContext.DEFAULT_PERM, true); Assert.assertTrue(isDir(fc, f)); @@ -210,13 +211,13 @@ public class TestFcHdfsSetUMask { public void testMkdirRecursiveWithNonExistingDir(FsPermission umask, FsPermission expectedPerms, FsPermission expectedParentPerms) throws IOException { - Path f = getTestRootPath(fc, "NonExistant2/aDir"); + Path f = fileContextTestHelper.getTestRootPath(fc, "NonExistant2/aDir"); fc.setUMask(umask); fc.mkdir(f, FileContext.DEFAULT_PERM, true); Assert.assertTrue(isDir(fc, f)); Assert.assertEquals("permissions on directory are wrong", expectedPerms, fc.getFileStatus(f).getPermission()); - Path fParent = getTestRootPath(fc, "NonExistant2"); + Path fParent = fileContextTestHelper.getTestRootPath(fc, "NonExistant2"); Assert.assertEquals("permissions on parent directory are wrong", expectedParentPerms, fc.getFileStatus(fParent).getPermission()); } @@ -224,7 +225,7 @@ public class TestFcHdfsSetUMask { public void testCreateRecursiveWithExistingDir(FsPermission umask, FsPermission expectedPerms) throws IOException { - Path f = getTestRootPath(fc,"foo"); + Path f = fileContextTestHelper.getTestRootPath(fc,"foo"); fc.setUMask(umask); createFile(fc, f); Assert.assertTrue(isFile(fc, f)); @@ -236,8 +237,8 @@ public class TestFcHdfsSetUMask { public void testCreateRecursiveWithNonExistingDir(FsPermission umask, FsPermission expectedDirPerms, FsPermission expectedFilePerms) throws IOException { - Path f = getTestRootPath(fc,"NonExisting/foo"); - Path fParent = getTestRootPath(fc, "NonExisting"); + Path f = fileContextTestHelper.getTestRootPath(fc,"NonExisting/foo"); + Path fParent = fileContextTestHelper.getTestRootPath(fc, "NonExisting"); Assert.assertFalse(exists(fc, fParent)); fc.setUMask(umask); createFile(fc, f); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java index f42f64d7609..2828cd0998f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.fs; -import static org.apache.hadoop.fs.FileContextTestHelper.getAbsoluteTestRootDir; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -51,10 +50,10 @@ public class TestFcHdfsSymlink extends FileContextSymlinkBaseTest { ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL); } + private static FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); private static MiniDFSCluster cluster; private static WebHdfsFileSystem webhdfs; private static DistributedFileSystem dfs; - @Override protected String getScheme() { @@ -103,10 +102,10 @@ public class TestFcHdfsSymlink extends FileContextSymlinkBaseTest { @Test /** Access a file using a link that spans Hdfs to LocalFs */ public void testLinkAcrossFileSystems() throws IOException { - Path localDir = new Path("file://"+getAbsoluteTestRootDir(fc)+"/test"); - Path localFile = new Path("file://"+getAbsoluteTestRootDir(fc)+"/test/file"); - Path link = new Path(testBaseDir1(), "linkToFile"); FileContext localFc = FileContext.getLocalFSFileContext(); + Path localDir = new Path("file://"+fileContextTestHelper.getAbsoluteTestRootDir(localFc)+"/test"); + Path localFile = new Path("file://"+fileContextTestHelper.getAbsoluteTestRootDir(localFc)+"/test/file"); + Path link = new Path(testBaseDir1(), "linkToFile"); localFc.delete(localDir, true); localFc.mkdir(localDir, FileContext.DEFAULT_PERM, true); localFc.setWorkingDirectory(localDir); @@ -120,12 +119,12 @@ public class TestFcHdfsSymlink extends FileContextSymlinkBaseTest { @Test /** Test renaming a file across two file systems using a link */ public void testRenameAcrossFileSystemsViaLink() throws IOException { - Path localDir = new Path("file://"+getAbsoluteTestRootDir(fc)+"/test"); + FileContext localFc = FileContext.getLocalFSFileContext(); + Path localDir = new Path("file://"+fileContextTestHelper.getAbsoluteTestRootDir(localFc)+"/test"); Path hdfsFile = new Path(testBaseDir1(), "file"); Path link = new Path(testBaseDir1(), "link"); Path hdfsFileNew = new Path(testBaseDir1(), "fileNew"); Path hdfsFileNewViaLink = new Path(link, "fileNew"); - FileContext localFc = FileContext.getLocalFSFileContext(); localFc.delete(localDir, true); localFc.mkdir(localDir, FileContext.DEFAULT_PERM, true); localFc.setWorkingDirectory(localDir); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java index a4f2d5fe3f9..2b58a079254 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java @@ -19,7 +19,6 @@ package org.apache.hadoop.fs; import static org.apache.hadoop.fs.FileContextTestHelper.exists; -import static org.apache.hadoop.fs.FileContextTestHelper.getTestRootPath; import java.io.IOException; import java.net.URISyntaxException; @@ -42,6 +41,7 @@ import org.junit.Test; public class TestHDFSFileContextMainOperations extends FileContextMainOperationsBaseTest { + private static FileContextTestHelper fileContextTestHelper = new FileContextTestHelper(); private static MiniDFSCluster cluster; private static Path defaultWorkingDirectory; private static HdfsConfiguration CONF = new HdfsConfiguration(); @@ -101,6 +101,10 @@ public class TestHDFSFileContextMainOperations extends return e; } + private Path getTestRootPath(FileContext fc, String path) { + return fileContextTestHelper.getTestRootPath(fc, path); + } + @Test public void testOldRenameWithQuota() throws Exception { DistributedFileSystem fs = (DistributedFileSystem) cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java index 72e087b0e5a..a0722f7bc39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java @@ -25,6 +25,7 @@ import javax.security.auth.login.LoginException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -42,6 +43,11 @@ public class TestViewFileSystemAtHdfsRoot extends ViewFileSystemBaseTest { private static Configuration CONF = new Configuration(); private static FileSystem fHdfs; + @Override + protected FileSystemTestHelper createFileSystemHelper() { + return new FileSystemTestHelper("/tmp/TestViewFileSystemAtHdfsRoot"); + } + @BeforeClass public static void clusterSetupAtBegining() throws IOException, LoginException, URISyntaxException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 3062ae433ec..1426d76e2d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -48,6 +48,11 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { private FileSystem fsTarget2; Path targetTestRoot2; + @Override + protected FileSystemTestHelper createFileSystemHelper() { + return new FileSystemTestHelper("/tmp/TestViewFileSystemHdfs"); + } + @BeforeClass public static void clusterSetupAtBegining() throws IOException, LoginException, URISyntaxException { @@ -86,7 +91,7 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { // create the test root on local_fs fsTarget = fHdfs; fsTarget2 = fHdfs2; - targetTestRoot2 = FileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2); + targetTestRoot2 = new FileSystemTestHelper().getAbsoluteTestRootPath(fsTarget2); super.setUp(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java index 74d8bca614d..5db84315229 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java @@ -52,6 +52,7 @@ public class TestViewFsFileStatusHdfs { static final String testfilename = "/tmp/testFileStatusSerialziation"; static final String someFile = "/hdfstmp/someFileForTestGetFileChecksum"; + private static final FileSystemTestHelper fileSystemTestHelper = new FileSystemTestHelper(); private static MiniDFSCluster cluster; private static Path defaultWorkingDirectory; private static Configuration CONF = new Configuration(); @@ -79,7 +80,7 @@ public class TestViewFsFileStatusHdfs { @Test public void testFileStatusSerialziation() throws IOException, URISyntaxException { - long len = FileSystemTestHelper.createFile(fHdfs, testfilename); + long len = fileSystemTestHelper.createFile(fHdfs, testfilename); FileStatus stat = vfs.getFileStatus(new Path(testfilename)); assertEquals(len, stat.getLen()); // check serialization/deserialization @@ -95,8 +96,8 @@ public class TestViewFsFileStatusHdfs { @Test public void testGetFileChecksum() throws IOException, URISyntaxException { // Create two different files in HDFS - FileSystemTestHelper.createFile(fHdfs, someFile); - FileSystemTestHelper.createFile(fHdfs, FileSystemTestHelper + fileSystemTestHelper.createFile(fHdfs, someFile); + fileSystemTestHelper.createFile(fHdfs, fileSystemTestHelper .getTestRootPath(fHdfs, someFile + "other"), 1, 512); // Get checksum through ViewFS FileChecksum viewFSCheckSum = vfs.getFileChecksum( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java index 28808e049ef..3d9fc8328ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestFSMainOperationsWebHdfs.java @@ -17,9 +17,6 @@ */ package org.apache.hadoop.hdfs.web; -import static org.apache.hadoop.fs.FileSystemTestHelper.exists; -import static org.apache.hadoop.fs.FileSystemTestHelper.getTestRootPath; - import java.io.IOException; import java.net.URI; import java.security.PrivilegedExceptionAction; @@ -52,6 +49,12 @@ public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest { private static MiniDFSCluster cluster = null; private static Path defaultWorkingDirectory; + private static FileSystem fileSystem; + + @Override + protected FileSystem createFileSystem() throws Exception { + return fileSystem; + } @BeforeClass public static void setupCluster() { @@ -73,14 +76,14 @@ public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest { final UserGroupInformation current = UserGroupInformation.getCurrentUser(); final UserGroupInformation ugi = UserGroupInformation.createUserForTesting( current.getShortUserName() + "x", new String[]{"user"}); - fSys = ugi.doAs(new PrivilegedExceptionAction() { + fileSystem = ugi.doAs(new PrivilegedExceptionAction() { @Override public FileSystem run() throws Exception { return FileSystem.get(new URI(uri), conf); } }); - defaultWorkingDirectory = fSys.getWorkingDirectory(); + defaultWorkingDirectory = fileSystem.getWorkingDirectory(); } catch (Exception e) { throw new RuntimeException(e); } @@ -159,4 +162,4 @@ public class TestFSMainOperationsWebHdfs extends FSMainOperationsBaseTest { // also okay for HDFS. } } -} \ No newline at end of file +} diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 4179b7f1f7d..79fca9db808 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -36,6 +36,9 @@ _ _ + + 4 + UTF-8 UTF-8 @@ -1004,5 +1007,23 @@ + diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java index 7151134c5f2..4301a83a90e 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java @@ -56,14 +56,14 @@ public class TestDistCpViewFs { public static void setup() throws URISyntaxException{ try { Path fswd = FileSystem.get(getConf()).getWorkingDirectory(); - Configuration vConf = ViewFileSystemTestSetup.createConfig(); - ConfigUtil.addLink(vConf, "/usr", new URI(fswd.toString())); + Configuration vConf = ViewFileSystemTestSetup.createConfig(false); + ConfigUtil.addLink(vConf, "/usr", new URI(fswd.toString())); fs = FileSystem.get(FsConstants.VIEWFS_URI, vConf); fs.setWorkingDirectory(new Path("/usr")); listFile = new Path("target/tmp/listing").makeQualified(fs.getUri(), fs.getWorkingDirectory()); target = new Path("target/tmp/target").makeQualified(fs.getUri(), - fs.getWorkingDirectory()); + fs.getWorkingDirectory()); root = new Path("target/tmp").makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(); TestDistCpUtils.delete(fs, root);