Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-14428. s3a: mkdir appears to be broken. Contributed by Mingliang Liu

This commit is contained in:
Mingliang Liu 2017-05-24 14:44:27 -07:00
parent abdd609e51
commit 6aeda55bb8
2 changed files with 23 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
View File

@ -113,18 +113,25 @@ public abstract class AbstractContractMkdirTest extends AbstractFSContractTestBa
describe("verify mkdir slash handling");
FileSystem fs = getFileSystem();
// No trailing slash
assertTrue(fs.mkdirs(path("testmkdir/a")));
assertPathExists("mkdir without trailing slash failed",
path("testmkdir/a"));
// With trailing slash
assertTrue(fs.mkdirs(path("testmkdir/b/")));
assertPathExists("mkdir with trailing slash failed", path("testmkdir/b/"));
// Mismatched slashes
assertPathExists("check path existence without trailing slash failed",
path("testmkdir/b"));
final Path[] paths = new Path[] {
path("testMkdirSlashHandling/a"), // w/o trailing slash
path("testMkdirSlashHandling/b/"), // w/ trailing slash
// unqualified w/o trailing slash
new Path(getContract().getTestPath() + "/testMkdirSlashHandling/c"),
// unqualified w/ trailing slash
new Path(getContract().getTestPath() + "/testMkdirSlashHandling/d/"),
// unqualified w/ multiple trailing slashes
new Path(getContract().getTestPath() + "/testMkdirSlashHandling/e///")
};
for (Path path : paths) {
assertTrue(fs.mkdirs(path));
assertPathExists(path + " does not exist after mkdirs", path);
assertIsDirectory(path);
if (path.toString().endsWith("/")) {
String s = path.toString().substring(0, path.toString().length() - 1);
assertIsDirectory(new Path(s));
}
}
}
@Test

5
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -1587,7 +1587,9 @@ public class S3AFileSystem extends FileSystem {
String key = pathToKey(f);
createFakeDirectory(key);
deleteUnnecessaryFakeDirectories(f.getParent());
// this is complicated because getParent(a/b/c/) returns a/b/c, but
// we want a/b. See HADOOP-14428 for more details.
deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
return true;
}
}
@ -1971,6 +1973,7 @@ public class S3AFileSystem extends FileSystem {
while (!path.isRoot()) {
String key = pathToKey(path);
key = (key.endsWith("/")) ? key : (key + "/");
LOG.trace("To delete unnecessary fake directory {} for {}", key, path);
keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key));
path = path.getParent();
}