HADOOP-13162. Consider reducing number of getFileStatus calls in S3AFileSystem.mkdirs. (Rajesh Balamohan via stevel)

This commit is contained in:
Steve Loughran 2016-06-01 14:17:18 +01:00
parent d749cf65e1
commit 5870611030
2 changed files with 73 additions and 3 deletions

View File

@ -20,14 +20,15 @@ package org.apache.hadoop.fs;
import java.io.IOException; import java.io.IOException;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import static org.apache.hadoop.fs.FileContextTestHelper.*; import static org.apache.hadoop.fs.FileContextTestHelper.*;
import org.apache.commons.logging.impl.Log4JLogger; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
/** /**
@ -116,7 +117,73 @@ public abstract class FileContextCreateMkdirBaseTest {
fc.mkdir(f, FileContext.DEFAULT_PERM, true); fc.mkdir(f, FileContext.DEFAULT_PERM, true);
Assert.assertTrue(isDir(fc, f)); Assert.assertTrue(isDir(fc, f));
} }
@Test
public void testMkdirsRecursiveWithExistingDir() throws IOException {
Path f = getTestRootPath(fc, "aDir/bDir/cDir");
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
assertIsDirectory(fc.getFileStatus(f));
assertIsDirectory(fc.getFileStatus(f.getParent()));
assertIsDirectory(fc.getFileStatus(f.getParent().getParent()));
}
@Test
public void testMkdirRecursiveWithExistingFile() throws IOException {
Path f = getTestRootPath(fc, "NonExistant3/aDir");
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
assertIsDirectory(fc.getFileStatus(f));
assertIsDirectory(fc.getFileStatus(f.getParent()));
// create a sample file
Path filePath = new Path(f.getParent(), "test.txt");
createFile(fc, filePath);
assertIsFile(filePath, fc.getFileStatus(filePath));
// try creating another folder which conflicts with filePath
Path dirPath = new Path(filePath, "bDir/cDir");
try {
fc.mkdir(dirPath, FileContext.DEFAULT_PERM, true);
Assert.fail("Mkdir for " + dirPath
+ " should have failed as a file was present");
} catch(IOException e) {
// failed as expected
}
}
@Test
public void testWithRename() throws IOException, InterruptedException {
Path root = getTestRootPath(fc);
Path f = new Path(root, "d1/d2/d3");
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
// create a sample file f.txt
Path fPath = new Path(root, "d1/d2/f.txt");
createFile(fc, fPath);
assertIsFile(fPath, fc.getFileStatus(fPath));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
// create a sample file f2.txt
Path f2Path = new Path(getTestRootPath(fc), "d1/d2/d3/f2.txt");
createFile(fc, f2Path);
assertIsFile(fPath, fc.getFileStatus(f2Path));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
//rename d1/d2/d3 d1/d4
fc.rename(new Path(root, "d1/d2/d3"), new Path(root, "d1/d4"));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d4")));
Path f2NewPath = new Path(root, "d1/d4/f2.txt");
assertIsFile(f2NewPath, fc.getFileStatus(f2NewPath));
}
/////////////////////// ///////////////////////
// Test Create // Test Create
//////////////////////// ////////////////////////

View File

@ -1088,6 +1088,9 @@ public class S3AFileSystem extends FileSystem {
do { do {
try { try {
FileStatus fileStatus = getFileStatus(fPart); FileStatus fileStatus = getFileStatus(fPart);
if (fileStatus.isDirectory()) {
break;
}
if (fileStatus.isFile()) { if (fileStatus.isFile()) {
throw new FileAlreadyExistsException(String.format( throw new FileAlreadyExistsException(String.format(
"Can't make directory for path '%s' since it is a file.", "Can't make directory for path '%s' since it is a file.",