Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-17105. S3AFS - Do not attempt to resolve symlinks in globStatus (#2113) 

Contributed by Jimmy Zuber.

Change-Id: I2f247c2d2ab4f38214073e55f5cfbaa15aeaeb11
This commit is contained in:
jimmy-zuber-amzn 2020-07-13 11:07:48 -07:00 committed by Steve Loughran
parent a51d72f0c6
commit 79fc58def3
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
2 changed files with 50 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -3980,6 +3980,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
/**
* Increments the statistic {@link Statistic#INVOCATION_GLOB_STATUS}.
* Override superclass so as to disable symlink resolution as symlinks
* are not supported by S3A.
* {@inheritDoc}
*/
@Override
@ -3988,9 +3990,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
}
/**
* Override superclass so as to disable symlink resolution and so avoid
* some calls to the FS which may have problems when the store is being
* inconsistent.
* Increments the statistic {@link Statistic#INVOCATION_GLOB_STATUS}.
* Override superclass so as to disable symlink resolution as symlinks
* are not supported by S3A.
* {@inheritDoc}
*/
@Override
@ -4002,7 +4004,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
return Globber.createGlobber(this)
.withPathPattern(pathPattern)
.withPathFiltern(filter)
.withResolveSymlinks(true)
.withResolveSymlinks(false)
.build()
.glob();
}

44
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
View File

@ -574,4 +574,48 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase {
}
}
@Test
public void testCostOfGlobStatus() throws Throwable {
describe("Test globStatus has expected cost");
S3AFileSystem fs = getFileSystem();
assume("Unguarded FS only", !fs.hasMetadataStore());
Path basePath = path("testCostOfGlobStatus/nextFolder/");
// create a bunch of files
int filesToCreate = 10;
for (int i = 0; i < filesToCreate; i++) {
try (FSDataOutputStream out = fs.create(basePath.suffix("/" + i))) {
verifyOperationCount(1, 1);
}
}
fs.globStatus(basePath.suffix("/*"));
// 2 head + 1 list from getFileStatus on path,
// plus 1 list to match the glob pattern
verifyOperationCount(2, 2);
}
@Test
public void testCostOfGlobStatusNoSymlinkResolution() throws Throwable {
describe("Test globStatus does not attempt to resolve symlinks");
S3AFileSystem fs = getFileSystem();
assume("Unguarded FS only", !fs.hasMetadataStore());
Path basePath = path("testCostOfGlobStatusNoSymlinkResolution/f/");
// create a single file, globStatus returning a single file on a pattern
// triggers attempts at symlinks resolution if configured
String fileName = "/notASymlinkDOntResolveMeLikeOne";
try (FSDataOutputStream out = fs.create(basePath.suffix(fileName))) {
verifyOperationCount(1, 1);
}
fs.globStatus(basePath.suffix("/*"));
// unguarded: 2 head + 1 list from getFileStatus on path,
// plus 1 list to match the glob pattern
// no additional operations from symlink resolution
verifyOperationCount(2, 2);
}
}