HADOOP-12774. s3a should use UGI.getCurrentUser.getShortname() for username. Contributed by Steve Loughran.
(cherry picked from commit 3372e94030
)
This commit is contained in:
parent
5b7cbb5a3c
commit
5c2f67bdae
|
@ -298,7 +298,7 @@ public class Listing {
|
||||||
// Skip over keys that are ourselves and old S3N _$folder$ files
|
// Skip over keys that are ourselves and old S3N _$folder$ files
|
||||||
if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) {
|
if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) {
|
||||||
FileStatus status = createFileStatus(keyPath, summary,
|
FileStatus status = createFileStatus(keyPath, summary,
|
||||||
owner.getDefaultBlockSize(keyPath));
|
owner.getDefaultBlockSize(keyPath), owner.getUsername());
|
||||||
LOG.debug("Adding: {}", status);
|
LOG.debug("Adding: {}", status);
|
||||||
stats.add(status);
|
stats.add(status);
|
||||||
added++;
|
added++;
|
||||||
|
@ -312,7 +312,8 @@ public class Listing {
|
||||||
for (String prefix : objects.getCommonPrefixes()) {
|
for (String prefix : objects.getCommonPrefixes()) {
|
||||||
Path keyPath = owner.keyToQualifiedPath(prefix);
|
Path keyPath = owner.keyToQualifiedPath(prefix);
|
||||||
if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) {
|
if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) {
|
||||||
FileStatus status = new S3AFileStatus(true, false, keyPath);
|
FileStatus status = new S3AFileStatus(false, keyPath,
|
||||||
|
owner.getUsername());
|
||||||
LOG.debug("Adding directory: {}", status);
|
LOG.debug("Adding directory: {}", status);
|
||||||
added++;
|
added++;
|
||||||
stats.add(status);
|
stats.add(status);
|
||||||
|
|
|
@ -33,28 +33,41 @@ import org.apache.hadoop.fs.Path;
|
||||||
public class S3AFileStatus extends FileStatus {
|
public class S3AFileStatus extends FileStatus {
|
||||||
private boolean isEmptyDirectory;
|
private boolean isEmptyDirectory;
|
||||||
|
|
||||||
// Directories
|
/**
|
||||||
public S3AFileStatus(boolean isdir, boolean isemptydir, Path path) {
|
* Create a directory status.
|
||||||
super(0, isdir, 1, 0, 0, path);
|
* @param isemptydir is this an empty directory?
|
||||||
|
* @param path the path
|
||||||
|
* @param owner the owner
|
||||||
|
*/
|
||||||
|
public S3AFileStatus(boolean isemptydir,
|
||||||
|
Path path,
|
||||||
|
String owner) {
|
||||||
|
super(0, true, 1, 0, 0, path);
|
||||||
isEmptyDirectory = isemptydir;
|
isEmptyDirectory = isemptydir;
|
||||||
|
setOwner(owner);
|
||||||
|
setGroup(owner);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Files
|
/**
|
||||||
|
* A simple file.
|
||||||
|
* @param length file length
|
||||||
|
* @param modification_time mod time
|
||||||
|
* @param path path
|
||||||
|
* @param blockSize block size
|
||||||
|
* @param owner owner
|
||||||
|
*/
|
||||||
public S3AFileStatus(long length, long modification_time, Path path,
|
public S3AFileStatus(long length, long modification_time, Path path,
|
||||||
long blockSize) {
|
long blockSize, String owner) {
|
||||||
super(length, false, 1, blockSize, modification_time, path);
|
super(length, false, 1, blockSize, modification_time, path);
|
||||||
isEmptyDirectory = false;
|
isEmptyDirectory = false;
|
||||||
|
setOwner(owner);
|
||||||
|
setGroup(owner);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isEmptyDirectory() {
|
public boolean isEmptyDirectory() {
|
||||||
return isEmptyDirectory;
|
return isEmptyDirectory;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getOwner() {
|
|
||||||
return System.getProperty("user.name");
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Compare if this object is equal to another object.
|
/** Compare if this object is equal to another object.
|
||||||
* @param o the object to be compared.
|
* @param o the object to be compared.
|
||||||
* @return true if two file status has the same path name; false if not.
|
* @return true if two file status has the same path name; false if not.
|
||||||
|
|
|
@ -86,6 +86,7 @@ import org.apache.hadoop.fs.RemoteIterator;
|
||||||
import org.apache.hadoop.fs.StorageStatistics;
|
import org.apache.hadoop.fs.StorageStatistics;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
|
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
|
||||||
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.util.Progressable;
|
import org.apache.hadoop.util.Progressable;
|
||||||
import org.apache.hadoop.util.ReflectionUtils;
|
import org.apache.hadoop.util.ReflectionUtils;
|
||||||
|
|
||||||
|
@ -119,6 +120,7 @@ public class S3AFileSystem extends FileSystem {
|
||||||
public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
|
public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
|
||||||
private URI uri;
|
private URI uri;
|
||||||
private Path workingDir;
|
private Path workingDir;
|
||||||
|
private String username;
|
||||||
private AmazonS3 s3;
|
private AmazonS3 s3;
|
||||||
private String bucket;
|
private String bucket;
|
||||||
private int maxKeys;
|
private int maxKeys;
|
||||||
|
@ -159,7 +161,9 @@ public class S3AFileSystem extends FileSystem {
|
||||||
instrumentation = new S3AInstrumentation(name);
|
instrumentation = new S3AInstrumentation(name);
|
||||||
|
|
||||||
uri = S3xLoginHelper.buildFSURI(name);
|
uri = S3xLoginHelper.buildFSURI(name);
|
||||||
workingDir = new Path("/user", System.getProperty("user.name"))
|
// Username is the current user at the time the FS was instantiated.
|
||||||
|
username = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||||
|
workingDir = new Path("/user", username)
|
||||||
.makeQualified(this.uri, this.getWorkingDirectory());
|
.makeQualified(this.uri, this.getWorkingDirectory());
|
||||||
|
|
||||||
bucket = name.getHost();
|
bucket = name.getHost();
|
||||||
|
@ -1386,6 +1390,14 @@ public class S3AFileSystem extends FileSystem {
|
||||||
return workingDir;
|
return workingDir;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the username of the FS.
|
||||||
|
* @return the short name of the user who instantiated the FS
|
||||||
|
*/
|
||||||
|
public String getUsername() {
|
||||||
|
return username;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* Make the given path and all non-existent parents into
|
* Make the given path and all non-existent parents into
|
||||||
|
@ -1477,14 +1489,14 @@ public class S3AFileSystem extends FileSystem {
|
||||||
|
|
||||||
if (objectRepresentsDirectory(key, meta.getContentLength())) {
|
if (objectRepresentsDirectory(key, meta.getContentLength())) {
|
||||||
LOG.debug("Found exact file: fake directory");
|
LOG.debug("Found exact file: fake directory");
|
||||||
return new S3AFileStatus(true, true,
|
return new S3AFileStatus(true, path, username);
|
||||||
path);
|
|
||||||
} else {
|
} else {
|
||||||
LOG.debug("Found exact file: normal file");
|
LOG.debug("Found exact file: normal file");
|
||||||
return new S3AFileStatus(meta.getContentLength(),
|
return new S3AFileStatus(meta.getContentLength(),
|
||||||
dateToLong(meta.getLastModified()),
|
dateToLong(meta.getLastModified()),
|
||||||
path,
|
path,
|
||||||
getDefaultBlockSize(path));
|
getDefaultBlockSize(path),
|
||||||
|
username);
|
||||||
}
|
}
|
||||||
} catch (AmazonServiceException e) {
|
} catch (AmazonServiceException e) {
|
||||||
if (e.getStatusCode() != 404) {
|
if (e.getStatusCode() != 404) {
|
||||||
|
@ -1502,7 +1514,7 @@ public class S3AFileSystem extends FileSystem {
|
||||||
|
|
||||||
if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
|
if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
|
||||||
LOG.debug("Found file (with /): fake directory");
|
LOG.debug("Found file (with /): fake directory");
|
||||||
return new S3AFileStatus(true, true, path);
|
return new S3AFileStatus(true, path, username);
|
||||||
} else {
|
} else {
|
||||||
LOG.warn("Found file (with /): real file? should not happen: {}",
|
LOG.warn("Found file (with /): real file? should not happen: {}",
|
||||||
key);
|
key);
|
||||||
|
@ -1510,7 +1522,8 @@ public class S3AFileSystem extends FileSystem {
|
||||||
return new S3AFileStatus(meta.getContentLength(),
|
return new S3AFileStatus(meta.getContentLength(),
|
||||||
dateToLong(meta.getLastModified()),
|
dateToLong(meta.getLastModified()),
|
||||||
path,
|
path,
|
||||||
getDefaultBlockSize(path));
|
getDefaultBlockSize(path),
|
||||||
|
username);
|
||||||
}
|
}
|
||||||
} catch (AmazonServiceException e) {
|
} catch (AmazonServiceException e) {
|
||||||
if (e.getStatusCode() != 404) {
|
if (e.getStatusCode() != 404) {
|
||||||
|
@ -1547,10 +1560,10 @@ public class S3AFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return new S3AFileStatus(true, false, path);
|
return new S3AFileStatus(false, path, username);
|
||||||
} else if (key.isEmpty()) {
|
} else if (key.isEmpty()) {
|
||||||
LOG.debug("Found root directory");
|
LOG.debug("Found root directory");
|
||||||
return new S3AFileStatus(true, true, path);
|
return new S3AFileStatus(true, path, username);
|
||||||
}
|
}
|
||||||
} catch (AmazonServiceException e) {
|
} catch (AmazonServiceException e) {
|
||||||
if (e.getStatusCode() != 404) {
|
if (e.getStatusCode() != 404) {
|
||||||
|
|
|
@ -241,17 +241,19 @@ public final class S3AUtils {
|
||||||
* @param keyPath path to entry
|
* @param keyPath path to entry
|
||||||
* @param summary summary from AWS
|
* @param summary summary from AWS
|
||||||
* @param blockSize block size to declare.
|
* @param blockSize block size to declare.
|
||||||
|
* @param owner owner of the file
|
||||||
* @return a status entry
|
* @return a status entry
|
||||||
*/
|
*/
|
||||||
public static S3AFileStatus createFileStatus(Path keyPath,
|
public static S3AFileStatus createFileStatus(Path keyPath,
|
||||||
S3ObjectSummary summary,
|
S3ObjectSummary summary,
|
||||||
long blockSize) {
|
long blockSize,
|
||||||
|
String owner) {
|
||||||
if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
|
if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
|
||||||
return new S3AFileStatus(true, true, keyPath);
|
return new S3AFileStatus(true, keyPath, owner);
|
||||||
} else {
|
} else {
|
||||||
return new S3AFileStatus(summary.getSize(),
|
return new S3AFileStatus(summary.getSize(),
|
||||||
dateToLong(summary.getLastModified()), keyPath,
|
dateToLong(summary.getLastModified()), keyPath,
|
||||||
blockSize);
|
blockSize, owner);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -44,8 +44,10 @@ import static org.junit.Assert.fail;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
|
import java.security.PrivilegedExceptionAction;
|
||||||
|
|
||||||
import org.apache.hadoop.security.ProviderUtils;
|
import org.apache.hadoop.security.ProviderUtils;
|
||||||
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.alias.CredentialProvider;
|
import org.apache.hadoop.security.alias.CredentialProvider;
|
||||||
import org.apache.hadoop.security.alias.CredentialProviderFactory;
|
import org.apache.hadoop.security.alias.CredentialProviderFactory;
|
||||||
import org.apache.hadoop.util.VersionInfo;
|
import org.apache.hadoop.util.VersionInfo;
|
||||||
|
@ -436,7 +438,7 @@ public class ITestS3AConfiguration {
|
||||||
dir1.mkdirs();
|
dir1.mkdirs();
|
||||||
dir2.mkdirs();
|
dir2.mkdirs();
|
||||||
conf = new Configuration();
|
conf = new Configuration();
|
||||||
conf.set(Constants.BUFFER_DIR, dir1 +", " + dir2);
|
conf.set(Constants.BUFFER_DIR, dir1 + ", " + dir2);
|
||||||
fs = S3ATestUtils.createTestFileSystem(conf);
|
fs = S3ATestUtils.createTestFileSystem(conf);
|
||||||
File tmp1 = fs.createTmpFileForWrite("out-", 1024, conf);
|
File tmp1 = fs.createTmpFileForWrite("out-", 1024, conf);
|
||||||
tmp1.delete();
|
tmp1.delete();
|
||||||
|
@ -446,6 +448,25 @@ public class ITestS3AConfiguration {
|
||||||
tmp1.getParent(), tmp2.getParent());
|
tmp1.getParent(), tmp2.getParent());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUsernameFromUGI() throws Throwable {
|
||||||
|
final String alice = "alice";
|
||||||
|
UserGroupInformation fakeUser =
|
||||||
|
UserGroupInformation.createUserForTesting(alice,
|
||||||
|
new String[]{"users", "administrators"});
|
||||||
|
conf = new Configuration();
|
||||||
|
fs = fakeUser.doAs(new PrivilegedExceptionAction<S3AFileSystem>() {
|
||||||
|
@Override
|
||||||
|
public S3AFileSystem run() throws Exception{
|
||||||
|
return S3ATestUtils.createTestFileSystem(conf);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
assertEquals("username", alice, fs.getUsername());
|
||||||
|
S3AFileStatus status = fs.getFileStatus(new Path("/"));
|
||||||
|
assertEquals("owner in " + status, alice, status.getOwner());
|
||||||
|
assertEquals("group in " + status, alice, status.getGroup());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads and returns a field from an object using reflection. If the field
|
* Reads and returns a field from an object using reflection. If the field
|
||||||
* cannot be found, is null, or is not the expected type, then this method
|
* cannot be found, is null, or is not the expected type, then this method
|
||||||
|
|
Loading…
Reference in New Issue