HADOOP-7552. svn merge -c 1164339 from trunk
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1164341 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
8f774dd5ab
commit
326cdada1f
|
@ -355,6 +355,9 @@ Release 0.23.0 - Unreleased
|
||||||
HADOOP-7604. Hadoop Auth examples pom in 0.23 point to 0.24 versions.
|
HADOOP-7604. Hadoop Auth examples pom in 0.23 point to 0.24 versions.
|
||||||
(mahadev)
|
(mahadev)
|
||||||
|
|
||||||
|
HADOOP-7552. FileUtil#fullyDelete doesn't throw IOE but lists it
|
||||||
|
in the throws clause. (eli)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
|
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
|
||||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.io.IOUtils;
|
import org.apache.hadoop.io.IOUtils;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
|
||||||
import org.apache.hadoop.util.Shell;
|
import org.apache.hadoop.util.Shell;
|
||||||
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
|
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
|
||||||
|
|
||||||
|
@ -88,7 +87,7 @@ public class FileUtil {
|
||||||
* (4) If dir is a normal directory, then dir and all its contents recursively
|
* (4) If dir is a normal directory, then dir and all its contents recursively
|
||||||
* are deleted.
|
* are deleted.
|
||||||
*/
|
*/
|
||||||
public static boolean fullyDelete(File dir) throws IOException {
|
public static boolean fullyDelete(File dir) {
|
||||||
if (dir.delete()) {
|
if (dir.delete()) {
|
||||||
// dir is (a) normal file, (b) symlink to a file, (c) empty directory or
|
// dir is (a) normal file, (b) symlink to a file, (c) empty directory or
|
||||||
// (d) symlink to a directory
|
// (d) symlink to a directory
|
||||||
|
@ -108,7 +107,7 @@ public class FileUtil {
|
||||||
* If dir is a symlink to a directory, all the contents of the actual
|
* If dir is a symlink to a directory, all the contents of the actual
|
||||||
* directory pointed to by dir will be deleted.
|
* directory pointed to by dir will be deleted.
|
||||||
*/
|
*/
|
||||||
public static boolean fullyDeleteContents(File dir) throws IOException {
|
public static boolean fullyDeleteContents(File dir) {
|
||||||
boolean deletionSucceeded = true;
|
boolean deletionSucceeded = true;
|
||||||
File contents[] = dir.listFiles();
|
File contents[] = dir.listFiles();
|
||||||
if (contents != null) {
|
if (contents != null) {
|
||||||
|
|
|
@ -228,10 +228,10 @@ public class RawLocalFileSystem extends FileSystem {
|
||||||
public FSDataOutputStream append(Path f, int bufferSize,
|
public FSDataOutputStream append(Path f, int bufferSize,
|
||||||
Progressable progress) throws IOException {
|
Progressable progress) throws IOException {
|
||||||
if (!exists(f)) {
|
if (!exists(f)) {
|
||||||
throw new FileNotFoundException("File " + f + " not found.");
|
throw new FileNotFoundException("File " + f + " not found");
|
||||||
}
|
}
|
||||||
if (getFileStatus(f).isDirectory()) {
|
if (getFileStatus(f).isDirectory()) {
|
||||||
throw new IOException("Cannot append to a diretory (=" + f + " ).");
|
throw new IOException("Cannot append to a diretory (=" + f + " )");
|
||||||
}
|
}
|
||||||
return new FSDataOutputStream(new BufferedOutputStream(
|
return new FSDataOutputStream(new BufferedOutputStream(
|
||||||
new LocalFSFileOutputStream(f, true), bufferSize), statistics);
|
new LocalFSFileOutputStream(f, true), bufferSize), statistics);
|
||||||
|
@ -242,7 +242,7 @@ public class RawLocalFileSystem extends FileSystem {
|
||||||
short replication, long blockSize, Progressable progress)
|
short replication, long blockSize, Progressable progress)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (exists(f) && !overwrite) {
|
if (exists(f) && !overwrite) {
|
||||||
throw new IOException("File already exists:"+f);
|
throw new IOException("File already exists: "+f);
|
||||||
}
|
}
|
||||||
Path parent = f.getParent();
|
Path parent = f.getParent();
|
||||||
if (parent != null && !mkdirs(parent)) {
|
if (parent != null && !mkdirs(parent)) {
|
||||||
|
@ -271,11 +271,18 @@ public class RawLocalFileSystem extends FileSystem {
|
||||||
return FileUtil.copy(this, src, this, dst, true, getConf());
|
return FileUtil.copy(this, src, this, dst, true, getConf());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete the given path to a file or directory.
|
||||||
|
* @param p the path to delete
|
||||||
|
* @param recursive to delete sub-directories
|
||||||
|
* @return true if the file or directory and all its contents were deleted
|
||||||
|
* @throws IOException if p is non-empty and recursive is false
|
||||||
|
*/
|
||||||
public boolean delete(Path p, boolean recursive) throws IOException {
|
public boolean delete(Path p, boolean recursive) throws IOException {
|
||||||
File f = pathToFile(p);
|
File f = pathToFile(p);
|
||||||
if (f.isFile()) {
|
if (f.isFile()) {
|
||||||
return f.delete();
|
return f.delete();
|
||||||
} else if ((!recursive) && f.isDirectory() &&
|
} else if (!recursive && f.isDirectory() &&
|
||||||
(FileUtil.listFiles(f).length != 0)) {
|
(FileUtil.listFiles(f).length != 0)) {
|
||||||
throw new IOException("Directory " + f.toString() + " is not empty");
|
throw new IOException("Directory " + f.toString() + " is not empty");
|
||||||
}
|
}
|
||||||
|
@ -287,7 +294,7 @@ public class RawLocalFileSystem extends FileSystem {
|
||||||
FileStatus[] results;
|
FileStatus[] results;
|
||||||
|
|
||||||
if (!localf.exists()) {
|
if (!localf.exists()) {
|
||||||
throw new FileNotFoundException("File " + f + " does not exist.");
|
throw new FileNotFoundException("File " + f + " does not exist");
|
||||||
}
|
}
|
||||||
if (localf.isFile()) {
|
if (localf.isFile()) {
|
||||||
return new FileStatus[] {
|
return new FileStatus[] {
|
||||||
|
@ -421,7 +428,7 @@ public class RawLocalFileSystem extends FileSystem {
|
||||||
if (path.exists()) {
|
if (path.exists()) {
|
||||||
return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(), this);
|
return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(), this);
|
||||||
} else {
|
} else {
|
||||||
throw new FileNotFoundException("File " + f + " does not exist.");
|
throw new FileNotFoundException("File " + f + " does not exist");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -158,10 +158,7 @@ public class RunJar {
|
||||||
|
|
||||||
Runtime.getRuntime().addShutdownHook(new Thread() {
|
Runtime.getRuntime().addShutdownHook(new Thread() {
|
||||||
public void run() {
|
public void run() {
|
||||||
try {
|
FileUtil.fullyDelete(workDir);
|
||||||
FileUtil.fullyDelete(workDir);
|
|
||||||
} catch (IOException e) {
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -17,16 +17,15 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.fs;
|
package org.apache.hadoop.fs;
|
||||||
|
|
||||||
import java.io.DataInputStream;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.io.IOUtils;
|
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper class for unit tests.
|
* Helper class for unit tests.
|
||||||
|
@ -143,23 +142,33 @@ public final class FileSystemTestHelper {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static String writeFile(FileSystem fileSys, Path name, int fileSize)
|
||||||
public static void writeFile(FileSystem fSys, Path path,byte b[])
|
throws IOException {
|
||||||
throws Exception {
|
final long seed = 0xDEADBEEFL;
|
||||||
FSDataOutputStream out =
|
// Create and write a file that contains three blocks of data
|
||||||
fSys.create(path);
|
FSDataOutputStream stm = fileSys.create(name);
|
||||||
out.write(b);
|
byte[] buffer = new byte[fileSize];
|
||||||
out.close();
|
Random rand = new Random(seed);
|
||||||
|
rand.nextBytes(buffer);
|
||||||
|
stm.write(buffer);
|
||||||
|
stm.close();
|
||||||
|
return new String(buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static byte[] readFile(FileSystem fSys, Path path, int len )
|
static String readFile(FileSystem fs, Path name, int buflen)
|
||||||
throws Exception {
|
throws IOException {
|
||||||
DataInputStream dis = fSys.open(path);
|
byte[] b = new byte[buflen];
|
||||||
byte[] buffer = new byte[len];
|
int offset = 0;
|
||||||
IOUtils.readFully(dis, buffer, 0, len);
|
FSDataInputStream in = fs.open(name);
|
||||||
dis.close();
|
for (int remaining, n;
|
||||||
return buffer;
|
(remaining = b.length - offset) > 0 && (n = in.read(b, offset, remaining)) != -1;
|
||||||
|
offset += n);
|
||||||
|
assertEquals(offset, Math.min(b.length, in.getPos()));
|
||||||
|
in.close();
|
||||||
|
String s = new String(b, 0, offset);
|
||||||
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static FileStatus containsPath(FileSystem fSys, Path path,
|
public static FileStatus containsPath(FileSystem fSys, Path path,
|
||||||
FileStatus[] dirList)
|
FileStatus[] dirList)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
|
@ -18,10 +18,9 @@
|
||||||
|
|
||||||
package org.apache.hadoop.fs;
|
package org.apache.hadoop.fs;
|
||||||
|
|
||||||
import java.net.URI;
|
|
||||||
|
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
|
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import junit.framework.TestCase;
|
import junit.framework.TestCase;
|
||||||
|
|
||||||
|
@ -56,13 +55,13 @@ public class TestChecksumFileSystem extends TestCase {
|
||||||
|
|
||||||
// Exercise some boundary cases - a divisor of the chunk size
|
// Exercise some boundary cases - a divisor of the chunk size
|
||||||
// the chunk size, 2x chunk size, and +/-1 around these.
|
// the chunk size, 2x chunk size, and +/-1 around these.
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 128);
|
readFile(localFs, testPath, 128);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 511);
|
readFile(localFs, testPath, 511);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 512);
|
readFile(localFs, testPath, 512);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 513);
|
readFile(localFs, testPath, 513);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 1023);
|
readFile(localFs, testPath, 1023);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 1024);
|
readFile(localFs, testPath, 1024);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 1025);
|
readFile(localFs, testPath, 1025);
|
||||||
|
|
||||||
localFs.delete(localFs.getChecksumFile(testPath), true);
|
localFs.delete(localFs.getChecksumFile(testPath), true);
|
||||||
assertTrue("checksum deleted", !localFs.exists(localFs.getChecksumFile(testPath)));
|
assertTrue("checksum deleted", !localFs.exists(localFs.getChecksumFile(testPath)));
|
||||||
|
@ -74,7 +73,7 @@ public class TestChecksumFileSystem extends TestCase {
|
||||||
|
|
||||||
boolean errorRead = false;
|
boolean errorRead = false;
|
||||||
try {
|
try {
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 1024);
|
readFile(localFs, testPath, 1024);
|
||||||
}catch(ChecksumException ie) {
|
}catch(ChecksumException ie) {
|
||||||
errorRead = true;
|
errorRead = true;
|
||||||
}
|
}
|
||||||
|
@ -83,7 +82,7 @@ public class TestChecksumFileSystem extends TestCase {
|
||||||
//now setting verify false, the read should succeed
|
//now setting verify false, the read should succeed
|
||||||
try {
|
try {
|
||||||
localFs.setVerifyChecksum(false);
|
localFs.setVerifyChecksum(false);
|
||||||
String str = TestLocalFileSystem.readFile(localFs, testPath, 1024);
|
String str = readFile(localFs, testPath, 1024).toString();
|
||||||
assertTrue("read", "testing".equals(str));
|
assertTrue("read", "testing".equals(str));
|
||||||
} finally {
|
} finally {
|
||||||
// reset for other tests
|
// reset for other tests
|
||||||
|
@ -104,13 +103,13 @@ public class TestChecksumFileSystem extends TestCase {
|
||||||
|
|
||||||
// Exercise some boundary cases - a divisor of the chunk size
|
// Exercise some boundary cases - a divisor of the chunk size
|
||||||
// the chunk size, 2x chunk size, and +/-1 around these.
|
// the chunk size, 2x chunk size, and +/-1 around these.
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 128);
|
readFile(localFs, testPath, 128);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 511);
|
readFile(localFs, testPath, 511);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 512);
|
readFile(localFs, testPath, 512);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 513);
|
readFile(localFs, testPath, 513);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 1023);
|
readFile(localFs, testPath, 1023);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 1024);
|
readFile(localFs, testPath, 1024);
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 1025);
|
readFile(localFs, testPath, 1025);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -140,7 +139,7 @@ public class TestChecksumFileSystem extends TestCase {
|
||||||
|
|
||||||
// Now reading the file should fail with a ChecksumException
|
// Now reading the file should fail with a ChecksumException
|
||||||
try {
|
try {
|
||||||
TestLocalFileSystem.readFile(localFs, testPath, 1024);
|
readFile(localFs, testPath, 1024);
|
||||||
fail("Did not throw a ChecksumException when reading truncated " +
|
fail("Did not throw a ChecksumException when reading truncated " +
|
||||||
"crc file");
|
"crc file");
|
||||||
} catch(ChecksumException ie) {
|
} catch(ChecksumException ie) {
|
||||||
|
@ -149,7 +148,7 @@ public class TestChecksumFileSystem extends TestCase {
|
||||||
// telling it not to verify checksums, should avoid issue.
|
// telling it not to verify checksums, should avoid issue.
|
||||||
try {
|
try {
|
||||||
localFs.setVerifyChecksum(false);
|
localFs.setVerifyChecksum(false);
|
||||||
String str = TestLocalFileSystem.readFile(localFs, testPath, 1024);
|
String str = readFile(localFs, testPath, 1024).toString();
|
||||||
assertTrue("read", "testing truncation".equals(str));
|
assertTrue("read", "testing truncation".equals(str));
|
||||||
} finally {
|
} finally {
|
||||||
// reset for other tests
|
// reset for other tests
|
||||||
|
|
|
@ -29,7 +29,7 @@ public class TestDU extends TestCase {
|
||||||
final static private File DU_DIR = new File(
|
final static private File DU_DIR = new File(
|
||||||
System.getProperty("test.build.data","/tmp"), "dutmp");
|
System.getProperty("test.build.data","/tmp"), "dutmp");
|
||||||
|
|
||||||
public void setUp() throws IOException {
|
public void setUp() {
|
||||||
FileUtil.fullyDelete(DU_DIR);
|
FileUtil.fullyDelete(DU_DIR);
|
||||||
assertTrue(DU_DIR.mkdirs());
|
assertTrue(DU_DIR.mkdirs());
|
||||||
}
|
}
|
||||||
|
|
|
@ -98,7 +98,7 @@ public class TestHardLink {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void setupClean() throws IOException {
|
public static void setupClean() {
|
||||||
//delete source and target directories if they exist
|
//delete source and target directories if they exist
|
||||||
FileUtil.fullyDelete(src);
|
FileUtil.fullyDelete(src);
|
||||||
FileUtil.fullyDelete(tgt_one);
|
FileUtil.fullyDelete(tgt_one);
|
||||||
|
|
|
@ -18,37 +18,23 @@
|
||||||
package org.apache.hadoop.fs;
|
package org.apache.hadoop.fs;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import junit.framework.*;
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class tests the local file system via the FileSystem abstraction.
|
* This class tests the local file system via the FileSystem abstraction.
|
||||||
*/
|
*/
|
||||||
public class TestLocalFileSystem extends TestCase {
|
public class TestLocalFileSystem {
|
||||||
private static String TEST_ROOT_DIR
|
private static String TEST_ROOT_DIR
|
||||||
= System.getProperty("test.build.data","build/test/data/work-dir/localfs");
|
= System.getProperty("test.build.data","build/test/data/work-dir/localfs");
|
||||||
|
|
||||||
|
private Configuration conf;
|
||||||
static void writeFile(FileSystem fs, Path name) throws IOException {
|
private FileSystem fileSys;
|
||||||
FSDataOutputStream stm = fs.create(name);
|
|
||||||
stm.writeBytes("42\n");
|
|
||||||
stm.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
static String readFile(FileSystem fs, Path name, int buflen) throws IOException {
|
|
||||||
byte[] b = new byte[buflen];
|
|
||||||
int offset = 0;
|
|
||||||
FSDataInputStream in = fs.open(name);
|
|
||||||
for(int remaining, n;
|
|
||||||
(remaining = b.length - offset) > 0 && (n = in.read(b, offset, remaining)) != -1;
|
|
||||||
offset += n);
|
|
||||||
assertEquals(offset, Math.min(b.length, in.getPos()));
|
|
||||||
in.close();
|
|
||||||
|
|
||||||
String s = new String(b, 0, offset);
|
|
||||||
System.out.println("s=" + s);
|
|
||||||
return s;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void cleanupFile(FileSystem fs, Path name) throws IOException {
|
private void cleanupFile(FileSystem fs, Path name) throws IOException {
|
||||||
assertTrue(fs.exists(name));
|
assertTrue(fs.exists(name));
|
||||||
|
@ -56,12 +42,18 @@ public class TestLocalFileSystem extends TestCase {
|
||||||
assertTrue(!fs.exists(name));
|
assertTrue(!fs.exists(name));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setup() throws IOException {
|
||||||
|
conf = new Configuration();
|
||||||
|
fileSys = FileSystem.getLocal(conf);
|
||||||
|
fileSys.delete(new Path(TEST_ROOT_DIR), true);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test the capability of setting the working directory.
|
* Test the capability of setting the working directory.
|
||||||
*/
|
*/
|
||||||
|
@Test
|
||||||
public void testWorkingDirectory() throws IOException {
|
public void testWorkingDirectory() throws IOException {
|
||||||
Configuration conf = new Configuration();
|
|
||||||
FileSystem fileSys = FileSystem.getLocal(conf);
|
|
||||||
Path origDir = fileSys.getWorkingDirectory();
|
Path origDir = fileSys.getWorkingDirectory();
|
||||||
Path subdir = new Path(TEST_ROOT_DIR, "new");
|
Path subdir = new Path(TEST_ROOT_DIR, "new");
|
||||||
try {
|
try {
|
||||||
|
@ -85,7 +77,7 @@ public class TestLocalFileSystem extends TestCase {
|
||||||
// create files and manipulate them.
|
// create files and manipulate them.
|
||||||
Path file1 = new Path("file1");
|
Path file1 = new Path("file1");
|
||||||
Path file2 = new Path("sub/file2");
|
Path file2 = new Path("sub/file2");
|
||||||
writeFile(fileSys, file1);
|
String contents = writeFile(fileSys, file1, 1);
|
||||||
fileSys.copyFromLocalFile(file1, file2);
|
fileSys.copyFromLocalFile(file1, file2);
|
||||||
assertTrue(fileSys.exists(file1));
|
assertTrue(fileSys.exists(file1));
|
||||||
assertTrue(fileSys.isFile(file1));
|
assertTrue(fileSys.isFile(file1));
|
||||||
|
@ -103,11 +95,10 @@ public class TestLocalFileSystem extends TestCase {
|
||||||
InputStream stm = fileSys.open(file1);
|
InputStream stm = fileSys.open(file1);
|
||||||
byte[] buffer = new byte[3];
|
byte[] buffer = new byte[3];
|
||||||
int bytesRead = stm.read(buffer, 0, 3);
|
int bytesRead = stm.read(buffer, 0, 3);
|
||||||
assertEquals("42\n", new String(buffer, 0, bytesRead));
|
assertEquals(contents, new String(buffer, 0, bytesRead));
|
||||||
stm.close();
|
stm.close();
|
||||||
} finally {
|
} finally {
|
||||||
fileSys.setWorkingDirectory(origDir);
|
fileSys.setWorkingDirectory(origDir);
|
||||||
fileSys.delete(subdir, true);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -115,6 +106,7 @@ public class TestLocalFileSystem extends TestCase {
|
||||||
* test Syncable interface on raw local file system
|
* test Syncable interface on raw local file system
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
|
@Test
|
||||||
public void testSyncable() throws IOException {
|
public void testSyncable() throws IOException {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
FileSystem fs = FileSystem.getLocal(conf).getRawFileSystem();
|
FileSystem fs = FileSystem.getLocal(conf).getRawFileSystem();
|
||||||
|
@ -148,12 +140,13 @@ public class TestLocalFileSystem extends TestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
public void testCopy() throws IOException {
|
public void testCopy() throws IOException {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
LocalFileSystem fs = FileSystem.getLocal(conf);
|
LocalFileSystem fs = FileSystem.getLocal(conf);
|
||||||
Path src = new Path(TEST_ROOT_DIR, "dingo");
|
Path src = new Path(TEST_ROOT_DIR, "dingo");
|
||||||
Path dst = new Path(TEST_ROOT_DIR, "yak");
|
Path dst = new Path(TEST_ROOT_DIR, "yak");
|
||||||
writeFile(fs, src);
|
writeFile(fs, src, 1);
|
||||||
assertTrue(FileUtil.copy(fs, src, fs, dst, true, false, conf));
|
assertTrue(FileUtil.copy(fs, src, fs, dst, true, false, conf));
|
||||||
assertTrue(!fs.exists(src) && fs.exists(dst));
|
assertTrue(!fs.exists(src) && fs.exists(dst));
|
||||||
assertTrue(FileUtil.copy(fs, dst, fs, src, false, false, conf));
|
assertTrue(FileUtil.copy(fs, dst, fs, src, false, false, conf));
|
||||||
|
@ -170,9 +163,12 @@ public class TestLocalFileSystem extends TestCase {
|
||||||
try {
|
try {
|
||||||
FileUtil.copy(fs, dst, fs, src, true, true, conf);
|
FileUtil.copy(fs, dst, fs, src, true, true, conf);
|
||||||
fail("Failed to detect existing dir");
|
fail("Failed to detect existing dir");
|
||||||
} catch (IOException e) { }
|
} catch (IOException e) {
|
||||||
|
// Expected
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
public void testHomeDirectory() throws IOException {
|
public void testHomeDirectory() throws IOException {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
FileSystem fileSys = FileSystem.getLocal(conf);
|
FileSystem fileSys = FileSystem.getLocal(conf);
|
||||||
|
@ -182,16 +178,18 @@ public class TestLocalFileSystem extends TestCase {
|
||||||
assertEquals(home, fsHome);
|
assertEquals(home, fsHome);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
public void testPathEscapes() throws IOException {
|
public void testPathEscapes() throws IOException {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
FileSystem fs = FileSystem.getLocal(conf);
|
FileSystem fs = FileSystem.getLocal(conf);
|
||||||
Path path = new Path(TEST_ROOT_DIR, "foo%bar");
|
Path path = new Path(TEST_ROOT_DIR, "foo%bar");
|
||||||
writeFile(fs, path);
|
writeFile(fs, path, 1);
|
||||||
FileStatus status = fs.getFileStatus(path);
|
FileStatus status = fs.getFileStatus(path);
|
||||||
assertEquals(path.makeQualified(fs), status.getPath());
|
assertEquals(path.makeQualified(fs), status.getPath());
|
||||||
cleanupFile(fs, path);
|
cleanupFile(fs, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
public void testMkdirs() throws IOException {
|
public void testMkdirs() throws IOException {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
LocalFileSystem fs = FileSystem.getLocal(conf);
|
LocalFileSystem fs = FileSystem.getLocal(conf);
|
||||||
|
@ -199,18 +197,40 @@ public class TestLocalFileSystem extends TestCase {
|
||||||
Path test_file = new Path(TEST_ROOT_DIR, "file1");
|
Path test_file = new Path(TEST_ROOT_DIR, "file1");
|
||||||
assertTrue(fs.mkdirs(test_dir));
|
assertTrue(fs.mkdirs(test_dir));
|
||||||
|
|
||||||
writeFile(fs, test_file);
|
writeFile(fs, test_file, 1);
|
||||||
// creating dir over a file
|
// creating dir over a file
|
||||||
Path bad_dir = new Path(test_file, "another_dir");
|
Path bad_dir = new Path(test_file, "another_dir");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
fs.mkdirs(bad_dir);
|
fs.mkdirs(bad_dir);
|
||||||
fail("Failed to detect existing file in path");
|
fail("Failed to detect existing file in path");
|
||||||
} catch (FileAlreadyExistsException e) { }
|
} catch (FileAlreadyExistsException e) {
|
||||||
|
// Expected
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
fs.mkdirs(null);
|
fs.mkdirs(null);
|
||||||
fail("Failed to detect null in mkdir arg");
|
fail("Failed to detect null in mkdir arg");
|
||||||
} catch (IllegalArgumentException e) { }
|
} catch (IllegalArgumentException e) {
|
||||||
|
// Expected
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test deleting a file, directory, and non-existent path */
|
||||||
|
@Test
|
||||||
|
public void testBasicDelete() throws IOException {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
LocalFileSystem fs = FileSystem.getLocal(conf);
|
||||||
|
Path dir1 = new Path(TEST_ROOT_DIR, "dir1");
|
||||||
|
Path file1 = new Path(TEST_ROOT_DIR, "file1");
|
||||||
|
Path file2 = new Path(TEST_ROOT_DIR+"/dir1", "file2");
|
||||||
|
Path file3 = new Path(TEST_ROOT_DIR, "does-not-exist");
|
||||||
|
assertTrue(fs.mkdirs(dir1));
|
||||||
|
writeFile(fs, file1, 1);
|
||||||
|
writeFile(fs, file2, 1);
|
||||||
|
assertFalse("Returned true deleting non-existant path",
|
||||||
|
fs.delete(file3));
|
||||||
|
assertTrue("Did not delete file", fs.delete(file1));
|
||||||
|
assertTrue("Did not delete non-empty dir", fs.delete(dir1));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,9 +19,9 @@ package org.apache.hadoop.fs;
|
||||||
|
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
|
import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
|
||||||
|
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.DataOutputStream;
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintStream;
|
import java.io.PrintStream;
|
||||||
|
@ -42,14 +42,6 @@ public class TestTrash extends TestCase {
|
||||||
new Path(new File(System.getProperty("test.build.data","/tmp")
|
new Path(new File(System.getProperty("test.build.data","/tmp")
|
||||||
).toURI().toString().replace(' ', '+'), "testTrash");
|
).toURI().toString().replace(' ', '+'), "testTrash");
|
||||||
|
|
||||||
protected static Path writeFile(FileSystem fs, Path f) throws IOException {
|
|
||||||
DataOutputStream out = fs.create(f);
|
|
||||||
out.writeBytes("dhruba: " + f);
|
|
||||||
out.close();
|
|
||||||
assertTrue(fs.exists(f));
|
|
||||||
return f;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected static Path mkdir(FileSystem fs, Path p) throws IOException {
|
protected static Path mkdir(FileSystem fs, Path p) throws IOException {
|
||||||
assertTrue(fs.mkdirs(p));
|
assertTrue(fs.mkdirs(p));
|
||||||
assertTrue(fs.exists(p));
|
assertTrue(fs.exists(p));
|
||||||
|
@ -139,7 +131,7 @@ public class TestTrash extends TestCase {
|
||||||
|
|
||||||
// Second, create a file in that directory.
|
// Second, create a file in that directory.
|
||||||
Path myFile = new Path(base, "test/mkdirs/myFile");
|
Path myFile = new Path(base, "test/mkdirs/myFile");
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
// Verify that expunge without Trash directory
|
// Verify that expunge without Trash directory
|
||||||
// won't throw Exception
|
// won't throw Exception
|
||||||
|
@ -176,7 +168,7 @@ public class TestTrash extends TestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify that we can recreate the file
|
// Verify that we can recreate the file
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
// Verify that we succeed in removing the file we re-created
|
// Verify that we succeed in removing the file we re-created
|
||||||
{
|
{
|
||||||
|
@ -194,7 +186,7 @@ public class TestTrash extends TestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify that we can recreate the file
|
// Verify that we can recreate the file
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
// Verify that we succeed in removing the whole directory
|
// Verify that we succeed in removing the whole directory
|
||||||
// along with the file inside it.
|
// along with the file inside it.
|
||||||
|
@ -234,7 +226,7 @@ public class TestTrash extends TestCase {
|
||||||
{
|
{
|
||||||
Path toErase = new Path(trashRoot, "toErase");
|
Path toErase = new Path(trashRoot, "toErase");
|
||||||
int retVal = -1;
|
int retVal = -1;
|
||||||
writeFile(trashRootFs, toErase);
|
writeFile(trashRootFs, toErase, 10);
|
||||||
try {
|
try {
|
||||||
retVal = shell.run(new String[] {"-rm", toErase.toString()});
|
retVal = shell.run(new String[] {"-rm", toErase.toString()});
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
@ -265,7 +257,7 @@ public class TestTrash extends TestCase {
|
||||||
|
|
||||||
// recreate directory and file
|
// recreate directory and file
|
||||||
mkdir(fs, myPath);
|
mkdir(fs, myPath);
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
// remove file first, then remove directory
|
// remove file first, then remove directory
|
||||||
{
|
{
|
||||||
|
@ -316,7 +308,7 @@ public class TestTrash extends TestCase {
|
||||||
|
|
||||||
// recreate directory and file
|
// recreate directory and file
|
||||||
mkdir(fs, myPath);
|
mkdir(fs, myPath);
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
// Verify that skip trash option really skips the trash for files (rm)
|
// Verify that skip trash option really skips the trash for files (rm)
|
||||||
{
|
{
|
||||||
|
@ -346,7 +338,7 @@ public class TestTrash extends TestCase {
|
||||||
|
|
||||||
// recreate directory and file
|
// recreate directory and file
|
||||||
mkdir(fs, myPath);
|
mkdir(fs, myPath);
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
// Verify that skip trash option really skips the trash for rmr
|
// Verify that skip trash option really skips the trash for rmr
|
||||||
{
|
{
|
||||||
|
@ -392,7 +384,7 @@ public class TestTrash extends TestCase {
|
||||||
for(int i=0;i<num_runs; i++) {
|
for(int i=0;i<num_runs; i++) {
|
||||||
|
|
||||||
//create file
|
//create file
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
// delete file
|
// delete file
|
||||||
try {
|
try {
|
||||||
|
@ -452,8 +444,7 @@ public class TestTrash extends TestCase {
|
||||||
lfs.delete(p, true);
|
lfs.delete(p, true);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
f = writeFile(lfs, f);
|
writeFile(lfs, f, 10);
|
||||||
|
|
||||||
FileSystem.closeAll();
|
FileSystem.closeAll();
|
||||||
FileSystem localFs = FileSystem.get(URI.create("file:///"), conf);
|
FileSystem localFs = FileSystem.get(URI.create("file:///"), conf);
|
||||||
Trash lTrash = new Trash(localFs, conf);
|
Trash lTrash = new Trash(localFs, conf);
|
||||||
|
@ -515,7 +506,7 @@ public class TestTrash extends TestCase {
|
||||||
while (true) {
|
while (true) {
|
||||||
// Create a file with a new name
|
// Create a file with a new name
|
||||||
Path myFile = new Path(TEST_DIR, "test/mkdirs/myFile" + fileIndex++);
|
Path myFile = new Path(TEST_DIR, "test/mkdirs/myFile" + fileIndex++);
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
// Delete the file to trash
|
// Delete the file to trash
|
||||||
String[] args = new String[2];
|
String[] args = new String[2];
|
||||||
|
@ -606,7 +597,7 @@ public class TestTrash extends TestCase {
|
||||||
int iters = 1000;
|
int iters = 1000;
|
||||||
for(int i=0;i<iters; i++) {
|
for(int i=0;i<iters; i++) {
|
||||||
|
|
||||||
writeFile(fs, myFile);
|
writeFile(fs, myFile, 10);
|
||||||
|
|
||||||
start = System.currentTimeMillis();
|
start = System.currentTimeMillis();
|
||||||
|
|
||||||
|
|
|
@ -50,7 +50,7 @@ public class TestNativeIO {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setupTestDir() throws IOException {
|
public void setupTestDir() {
|
||||||
FileUtil.fullyDelete(TEST_DIR);
|
FileUtil.fullyDelete(TEST_DIR);
|
||||||
TEST_DIR.mkdirs();
|
TEST_DIR.mkdirs();
|
||||||
}
|
}
|
||||||
|
|
|
@ -49,8 +49,7 @@ public class TestRunJar extends TestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
protected void tearDown()
|
protected void tearDown() {
|
||||||
throws Exception {
|
|
||||||
FileUtil.fullyDelete(TEST_ROOT_DIR);
|
FileUtil.fullyDelete(TEST_ROOT_DIR);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -84,6 +84,7 @@ public class TestListFilesInFileContext {
|
||||||
public static void testShutdown() throws Exception {
|
public static void testShutdown() throws Exception {
|
||||||
cluster.shutdown();
|
cluster.shutdown();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Test when input path is a file */
|
/** Test when input path is a file */
|
||||||
@Test
|
@Test
|
||||||
public void testFile() throws IOException {
|
public void testFile() throws IOException {
|
||||||
|
@ -199,4 +200,4 @@ public class TestListFilesInFileContext {
|
||||||
assertEquals(fc.makeQualified(FILE1), stat.getPath());
|
assertEquals(fc.makeQualified(FILE1), stat.getPath());
|
||||||
assertFalse(itor.hasNext());
|
assertFalse(itor.hasNext());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -59,7 +59,7 @@ public class TestJvmManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
public void tearDown() throws IOException {
|
public void tearDown() {
|
||||||
FileUtil.fullyDelete(TEST_DIR);
|
FileUtil.fullyDelete(TEST_DIR);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -45,7 +45,7 @@ public class TestLinuxTaskController extends TestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
protected void tearDown() throws Exception {
|
protected void tearDown() {
|
||||||
FileUtil.fullyDelete(testDir);
|
FileUtil.fullyDelete(testDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -33,7 +33,7 @@ public class TestTaskOutputSize {
|
||||||
"/tmp"), "test");
|
"/tmp"), "test");
|
||||||
|
|
||||||
@After
|
@After
|
||||||
public void tearDown() throws Exception {
|
public void tearDown() {
|
||||||
FileUtil.fullyDelete(new File(rootDir.toString()));
|
FileUtil.fullyDelete(new File(rootDir.toString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -61,7 +61,7 @@ public class TestUserLogCleanup {
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
public void tearDown() throws IOException {
|
public void tearDown() {
|
||||||
FileUtil.fullyDelete(TaskLog.getUserLogDir());
|
FileUtil.fullyDelete(TaskLog.getUserLogDir());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -43,7 +43,7 @@ public class TestMRAsyncDiskService extends TestCase {
|
||||||
"test.build.data", "/tmp")).toString();
|
"test.build.data", "/tmp")).toString();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void setUp() throws Exception {
|
protected void setUp() {
|
||||||
FileUtil.fullyDelete(new File(TEST_ROOT_DIR));
|
FileUtil.fullyDelete(new File(TEST_ROOT_DIR));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -592,8 +592,7 @@ public class TestProcfsBasedProcessTree extends TestCase {
|
||||||
* @param procfsRootDir root directory to create.
|
* @param procfsRootDir root directory to create.
|
||||||
* @throws IOException if could not delete the procfs root directory
|
* @throws IOException if could not delete the procfs root directory
|
||||||
*/
|
*/
|
||||||
public static void setupProcfsRootDir(File procfsRootDir)
|
public static void setupProcfsRootDir(File procfsRootDir) {
|
||||||
throws IOException {
|
|
||||||
// cleanup any existing process root dir.
|
// cleanup any existing process root dir.
|
||||||
if (procfsRootDir.exists()) {
|
if (procfsRootDir.exists()) {
|
||||||
assertTrue(FileUtil.fullyDelete(procfsRootDir));
|
assertTrue(FileUtil.fullyDelete(procfsRootDir));
|
||||||
|
|
Loading…
Reference in New Issue