HDFS-10832. Propagate ACL bit and isEncrypted bit in HttpFS FileStatus permissions.
This commit is contained in:
Andrew Wang
parent
9f192cc5ac
commit
cba973f036
|
|
@ -38,6 +38,7 @@ import org.apache.hadoop.fs.permission.AclEntry;
|
|||
import org.apache.hadoop.fs.permission.AclStatus;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
|
||||
import org.apache.hadoop.lib.wsrs.EnumSetParam;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
|
|
@ -181,6 +182,8 @@ public class HttpFSFileSystem extends FileSystem
|
|||
public static final String ACL_ENTRIES_JSON = "entries";
|
||||
public static final String ACL_BIT_JSON = "aclBit";
|
||||
|
||||
public static final String ENC_BIT_JSON = "encBit";
|
||||
|
||||
public static final int HTTP_TEMPORARY_REDIRECT = 307;
|
||||
|
||||
private static final String HTTP_GET = "GET";
|
||||
|
|
@ -955,6 +958,21 @@ public class HttpFSFileSystem extends FileSystem
|
|||
return createAclStatus(json);
|
||||
}
|
||||
|
||||
/** Convert a string to a FsPermission object. */
|
||||
static FsPermission toFsPermission(JSONObject json) {
|
||||
final String s = (String) json.get(PERMISSION_JSON);
|
||||
final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON);
|
||||
final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON);
|
||||
FsPermission perm = new FsPermission(Short.parseShort(s, 8));
|
||||
final boolean aBit = (aclBit != null) ? aclBit : false;
|
||||
final boolean eBit = (encBit != null) ? encBit : false;
|
||||
if (aBit || eBit) {
|
||||
return new FsPermissionExtension(perm, aBit, eBit);
|
||||
} else {
|
||||
return perm;
|
||||
}
|
||||
}
|
||||
|
||||
private FileStatus createFileStatus(Path parent, JSONObject json) {
|
||||
String pathSuffix = (String) json.get(PATH_SUFFIX_JSON);
|
||||
Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix);
|
||||
|
|
@ -962,8 +980,7 @@ public class HttpFSFileSystem extends FileSystem
|
|||
long len = (Long) json.get(LENGTH_JSON);
|
||||
String owner = (String) json.get(OWNER_JSON);
|
||||
String group = (String) json.get(GROUP_JSON);
|
||||
FsPermission permission =
|
||||
new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8));
|
||||
final FsPermission permission = toFsPermission(json);
|
||||
long aTime = (Long) json.get(ACCESS_TIME_JSON);
|
||||
long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
|
||||
long blockSize = (Long) json.get(BLOCK_SIZE_JSON);
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ import org.apache.hadoop.fs.http.client.HttpFSFileSystem;
|
|||
import org.apache.hadoop.fs.permission.AclEntry;
|
||||
import org.apache.hadoop.fs.permission.AclStatus;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.hdfs.protocol.AclException;
|
||||
import org.apache.hadoop.io.IOUtils;
|
||||
import org.apache.hadoop.lib.service.FileSystemAccess;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
|
@ -54,148 +53,59 @@ import java.util.Map.Entry;
|
|||
public class FSOperations {
|
||||
|
||||
/**
|
||||
* This class is used to group a FileStatus and an AclStatus together.
|
||||
* It's needed for the GETFILESTATUS and LISTSTATUS calls, which take
|
||||
* most info from the FileStatus and a wee bit from the AclStatus.
|
||||
* @param fileStatus a FileStatus object
|
||||
* @return JSON map suitable for wire transport
|
||||
*/
|
||||
private static class StatusPair {
|
||||
private FileStatus fileStatus;
|
||||
private AclStatus aclStatus;
|
||||
|
||||
/**
|
||||
* Simple constructor
|
||||
* @param fileStatus Existing FileStatus object
|
||||
* @param aclStatus Existing AclStatus object
|
||||
*/
|
||||
public StatusPair(FileStatus fileStatus, AclStatus aclStatus) {
|
||||
this.fileStatus = fileStatus;
|
||||
this.aclStatus = aclStatus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create one StatusPair by performing the underlying calls to
|
||||
* fs.getFileStatus and fs.getAclStatus
|
||||
* @param fs The FileSystem where 'path' lives
|
||||
* @param path The file/directory to query
|
||||
* @throws IOException
|
||||
*/
|
||||
public StatusPair(FileSystem fs, Path path) throws IOException {
|
||||
fileStatus = fs.getFileStatus(path);
|
||||
aclStatus = null;
|
||||
try {
|
||||
aclStatus = fs.getAclStatus(path);
|
||||
} catch (AclException e) {
|
||||
/*
|
||||
* The cause is almost certainly an "ACLS aren't enabled"
|
||||
* exception, so leave aclStatus at null and carry on.
|
||||
*/
|
||||
} catch (UnsupportedOperationException e) {
|
||||
/* Ditto above - this is the case for a local file system */
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a Map suitable for conversion into JSON format
|
||||
* @return The JSONish Map
|
||||
*/
|
||||
public Map<String,Object> toJson() {
|
||||
Map<String,Object> json = new LinkedHashMap<String,Object>();
|
||||
json.put(HttpFSFileSystem.FILE_STATUS_JSON, toJsonInner(true));
|
||||
return json;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return in inner part of the JSON for the status - used by both the
|
||||
* GETFILESTATUS and LISTSTATUS calls.
|
||||
* @param emptyPathSuffix Whether or not to include PATH_SUFFIX_JSON
|
||||
* @return The JSONish Map
|
||||
*/
|
||||
public Map<String,Object> toJsonInner(boolean emptyPathSuffix) {
|
||||
Map<String,Object> json = new LinkedHashMap<String,Object>();
|
||||
json.put(HttpFSFileSystem.PATH_SUFFIX_JSON,
|
||||
(emptyPathSuffix) ? "" : fileStatus.getPath().getName());
|
||||
json.put(HttpFSFileSystem.TYPE_JSON,
|
||||
HttpFSFileSystem.FILE_TYPE.getType(fileStatus).toString());
|
||||
json.put(HttpFSFileSystem.LENGTH_JSON, fileStatus.getLen());
|
||||
json.put(HttpFSFileSystem.OWNER_JSON, fileStatus.getOwner());
|
||||
json.put(HttpFSFileSystem.GROUP_JSON, fileStatus.getGroup());
|
||||
json.put(HttpFSFileSystem.PERMISSION_JSON,
|
||||
HttpFSFileSystem.permissionToString(fileStatus.getPermission()));
|
||||
json.put(HttpFSFileSystem.ACCESS_TIME_JSON, fileStatus.getAccessTime());
|
||||
json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON,
|
||||
fileStatus.getModificationTime());
|
||||
json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, fileStatus.getBlockSize());
|
||||
json.put(HttpFSFileSystem.REPLICATION_JSON, fileStatus.getReplication());
|
||||
if ( (aclStatus != null) && !(aclStatus.getEntries().isEmpty()) ) {
|
||||
json.put(HttpFSFileSystem.ACL_BIT_JSON,true);
|
||||
}
|
||||
return json;
|
||||
}
|
||||
private static Map<String, Object> toJson(FileStatus fileStatus) {
|
||||
Map<String, Object> json = new LinkedHashMap<>();
|
||||
json.put(HttpFSFileSystem.FILE_STATUS_JSON, toJsonInner(fileStatus, true));
|
||||
return json;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple class used to contain and operate upon a list of StatusPair
|
||||
* objects. Used by LISTSTATUS.
|
||||
* @param fileStatuses list of FileStatus objects
|
||||
* @return JSON map suitable for wire transport
|
||||
*/
|
||||
private static class StatusPairs {
|
||||
private StatusPair[] statusPairs;
|
||||
|
||||
/**
|
||||
* Construct a list of StatusPair objects
|
||||
* @param fs The FileSystem where 'path' lives
|
||||
* @param path The directory to query
|
||||
* @param filter A possible filter for entries in the directory
|
||||
* @throws IOException
|
||||
*/
|
||||
public StatusPairs(FileSystem fs, Path path, PathFilter filter)
|
||||
throws IOException {
|
||||
/* Grab all the file statuses at once in an array */
|
||||
FileStatus[] fileStatuses = fs.listStatus(path, filter);
|
||||
|
||||
/* We'll have an array of StatusPairs of the same length */
|
||||
AclStatus aclStatus = null;
|
||||
statusPairs = new StatusPair[fileStatuses.length];
|
||||
|
||||
/*
|
||||
* For each FileStatus, attempt to acquire an AclStatus. If the
|
||||
* getAclStatus throws an exception, we assume that ACLs are turned
|
||||
* off entirely and abandon the attempt.
|
||||
*/
|
||||
boolean useAcls = true; // Assume ACLs work until proven otherwise
|
||||
for (int i = 0; i < fileStatuses.length; i++) {
|
||||
if (useAcls) {
|
||||
try {
|
||||
aclStatus = fs.getAclStatus(fileStatuses[i].getPath());
|
||||
} catch (AclException e) {
|
||||
/* Almost certainly due to an "ACLs not enabled" exception */
|
||||
aclStatus = null;
|
||||
useAcls = false;
|
||||
} catch (UnsupportedOperationException e) {
|
||||
/* Ditto above - this is the case for a local file system */
|
||||
aclStatus = null;
|
||||
useAcls = false;
|
||||
}
|
||||
}
|
||||
statusPairs[i] = new StatusPair(fileStatuses[i], aclStatus);
|
||||
}
|
||||
@SuppressWarnings({"unchecked"})
|
||||
private static Map<String, Object> toJson(FileStatus[] fileStatuses) {
|
||||
Map<String, Object> json = new LinkedHashMap<>();
|
||||
Map<String, Object> inner = new LinkedHashMap<>();
|
||||
JSONArray statuses = new JSONArray();
|
||||
for (FileStatus f : fileStatuses) {
|
||||
statuses.add(toJsonInner(f, false));
|
||||
}
|
||||
inner.put(HttpFSFileSystem.FILE_STATUS_JSON, statuses);
|
||||
json.put(HttpFSFileSystem.FILE_STATUSES_JSON, inner);
|
||||
return json;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a Map suitable for conversion into JSON.
|
||||
* @return A JSONish Map
|
||||
*/
|
||||
@SuppressWarnings({"unchecked"})
|
||||
public Map<String,Object> toJson() {
|
||||
Map<String,Object> json = new LinkedHashMap<String,Object>();
|
||||
Map<String,Object> inner = new LinkedHashMap<String,Object>();
|
||||
JSONArray statuses = new JSONArray();
|
||||
for (StatusPair s : statusPairs) {
|
||||
statuses.add(s.toJsonInner(false));
|
||||
}
|
||||
inner.put(HttpFSFileSystem.FILE_STATUS_JSON, statuses);
|
||||
json.put(HttpFSFileSystem.FILE_STATUSES_JSON, inner);
|
||||
return json;
|
||||
/**
|
||||
* Not meant to be called directly except by the other toJson functions.
|
||||
*/
|
||||
private static Map<String, Object> toJsonInner(FileStatus fileStatus,
|
||||
boolean emptyPathSuffix) {
|
||||
Map<String, Object> json = new LinkedHashMap<String, Object>();
|
||||
json.put(HttpFSFileSystem.PATH_SUFFIX_JSON,
|
||||
(emptyPathSuffix) ? "" : fileStatus.getPath().getName());
|
||||
json.put(HttpFSFileSystem.TYPE_JSON,
|
||||
HttpFSFileSystem.FILE_TYPE.getType(fileStatus).toString());
|
||||
json.put(HttpFSFileSystem.LENGTH_JSON, fileStatus.getLen());
|
||||
json.put(HttpFSFileSystem.OWNER_JSON, fileStatus.getOwner());
|
||||
json.put(HttpFSFileSystem.GROUP_JSON, fileStatus.getGroup());
|
||||
json.put(HttpFSFileSystem.PERMISSION_JSON,
|
||||
HttpFSFileSystem.permissionToString(fileStatus.getPermission()));
|
||||
json.put(HttpFSFileSystem.ACCESS_TIME_JSON, fileStatus.getAccessTime());
|
||||
json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON,
|
||||
fileStatus.getModificationTime());
|
||||
json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, fileStatus.getBlockSize());
|
||||
json.put(HttpFSFileSystem.REPLICATION_JSON, fileStatus.getReplication());
|
||||
if (fileStatus.getPermission().getAclBit()) {
|
||||
json.put(HttpFSFileSystem.ACL_BIT_JSON, true);
|
||||
}
|
||||
if (fileStatus.getPermission().getEncryptedBit()) {
|
||||
json.put(HttpFSFileSystem.ENC_BIT_JSON, true);
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
/** Converts an <code>AclStatus</code> object into a JSON object.
|
||||
|
|
@ -637,8 +547,8 @@ public class FSOperations {
|
|||
*/
|
||||
@Override
|
||||
public Map execute(FileSystem fs) throws IOException {
|
||||
StatusPair sp = new StatusPair(fs, path);
|
||||
return sp.toJson();
|
||||
FileStatus status = fs.getFileStatus(path);
|
||||
return toJson(status);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -703,8 +613,8 @@ public class FSOperations {
|
|||
*/
|
||||
@Override
|
||||
public Map execute(FileSystem fs) throws IOException {
|
||||
StatusPairs sp = new StatusPairs(fs, path, filter);
|
||||
return sp.toJson();
|
||||
FileStatus[] fileStatuses = fs.listStatus(path, filter);
|
||||
return toJson(fileStatuses);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ import org.apache.hadoop.test.HadoopUsersConfTestHelper;
|
|||
import org.apache.hadoop.test.TestDir;
|
||||
import org.apache.hadoop.test.TestDirHelper;
|
||||
import org.apache.hadoop.test.TestHdfs;
|
||||
import org.apache.hadoop.test.TestHdfsHelper;
|
||||
import org.apache.hadoop.test.TestJetty;
|
||||
import org.apache.hadoop.test.TestJettyHelper;
|
||||
import org.junit.Assert;
|
||||
|
|
@ -66,6 +67,11 @@ import java.util.Collection;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
@RunWith(value = Parameterized.class)
|
||||
public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
||||
|
||||
|
|
@ -81,9 +87,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
|
||||
private void createHttpFSServer() throws Exception {
|
||||
File homeDir = TestDirHelper.getTestDir();
|
||||
Assert.assertTrue(new File(homeDir, "conf").mkdir());
|
||||
Assert.assertTrue(new File(homeDir, "log").mkdir());
|
||||
Assert.assertTrue(new File(homeDir, "temp").mkdir());
|
||||
assertTrue(new File(homeDir, "conf").mkdir());
|
||||
assertTrue(new File(homeDir, "log").mkdir());
|
||||
assertTrue(new File(homeDir, "temp").mkdir());
|
||||
HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath());
|
||||
|
||||
File secretFile = new File(new File(homeDir, "conf"), "secret");
|
||||
|
|
@ -143,7 +149,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
Assert.assertNotNull(fs);
|
||||
URI uri = new URI(getScheme() + "://" +
|
||||
TestJettyHelper.getJettyURL().toURI().getAuthority());
|
||||
Assert.assertEquals(fs.getUri(), uri);
|
||||
assertEquals(fs.getUri(), uri);
|
||||
fs.close();
|
||||
}
|
||||
|
||||
|
|
@ -156,7 +162,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs.close();
|
||||
fs = getHttpFSFileSystem();
|
||||
InputStream is = fs.open(new Path(path.toUri().getPath()));
|
||||
Assert.assertEquals(is.read(), 1);
|
||||
assertEquals(is.read(), 1);
|
||||
is.close();
|
||||
fs.close();
|
||||
}
|
||||
|
|
@ -173,12 +179,12 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs = FileSystem.get(getProxiedFSConf());
|
||||
FileStatus status = fs.getFileStatus(path);
|
||||
if (!isLocalFS()) {
|
||||
Assert.assertEquals(status.getReplication(), 2);
|
||||
Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024);
|
||||
assertEquals(status.getReplication(), 2);
|
||||
assertEquals(status.getBlockSize(), 100 * 1024 * 1024);
|
||||
}
|
||||
Assert.assertEquals(status.getPermission(), permission);
|
||||
assertEquals(status.getPermission(), permission);
|
||||
InputStream is = fs.open(path);
|
||||
Assert.assertEquals(is.read(), 1);
|
||||
assertEquals(is.read(), 1);
|
||||
is.close();
|
||||
fs.close();
|
||||
}
|
||||
|
|
@ -216,9 +222,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs.close();
|
||||
fs = FileSystem.get(getProxiedFSConf());
|
||||
InputStream is = fs.open(path);
|
||||
Assert.assertEquals(is.read(), 1);
|
||||
Assert.assertEquals(is.read(), 2);
|
||||
Assert.assertEquals(is.read(), -1);
|
||||
assertEquals(is.read(), 1);
|
||||
assertEquals(is.read(), 2);
|
||||
assertEquals(is.read(), -1);
|
||||
is.close();
|
||||
fs.close();
|
||||
}
|
||||
|
|
@ -239,10 +245,10 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
final int newLength = blockSize;
|
||||
|
||||
boolean isReady = fs.truncate(file, newLength);
|
||||
Assert.assertTrue("Recovery is not expected.", isReady);
|
||||
assertTrue("Recovery is not expected.", isReady);
|
||||
|
||||
FileStatus fileStatus = fs.getFileStatus(file);
|
||||
Assert.assertEquals(fileStatus.getLen(), newLength);
|
||||
assertEquals(fileStatus.getLen(), newLength);
|
||||
AppendTestUtil.checkFullFile(fs, file, newLength, data, file.toString());
|
||||
|
||||
fs.close();
|
||||
|
|
@ -266,9 +272,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs.concat(path1, new Path[]{path2, path3});
|
||||
fs.close();
|
||||
fs = FileSystem.get(config);
|
||||
Assert.assertTrue(fs.exists(path1));
|
||||
Assert.assertFalse(fs.exists(path2));
|
||||
Assert.assertFalse(fs.exists(path3));
|
||||
assertTrue(fs.exists(path1));
|
||||
assertFalse(fs.exists(path2));
|
||||
assertFalse(fs.exists(path3));
|
||||
fs.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -284,8 +290,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs.rename(oldPath, newPath);
|
||||
fs.close();
|
||||
fs = FileSystem.get(getProxiedFSConf());
|
||||
Assert.assertFalse(fs.exists(oldPath));
|
||||
Assert.assertTrue(fs.exists(newPath));
|
||||
assertFalse(fs.exists(oldPath));
|
||||
assertTrue(fs.exists(newPath));
|
||||
fs.close();
|
||||
}
|
||||
|
||||
|
|
@ -299,8 +305,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs.mkdirs(foe);
|
||||
|
||||
FileSystem hoopFs = getHttpFSFileSystem();
|
||||
Assert.assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false));
|
||||
Assert.assertFalse(fs.exists(foo));
|
||||
assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false));
|
||||
assertFalse(fs.exists(foo));
|
||||
try {
|
||||
hoopFs.delete(new Path(bar.toUri().getPath()), false);
|
||||
Assert.fail();
|
||||
|
|
@ -308,13 +314,13 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
} catch (Exception ex) {
|
||||
Assert.fail();
|
||||
}
|
||||
Assert.assertTrue(fs.exists(bar));
|
||||
Assert.assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true));
|
||||
Assert.assertFalse(fs.exists(bar));
|
||||
assertTrue(fs.exists(bar));
|
||||
assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true));
|
||||
assertFalse(fs.exists(bar));
|
||||
|
||||
Assert.assertTrue(fs.exists(foe));
|
||||
Assert.assertTrue(hoopFs.delete(foe, true));
|
||||
Assert.assertFalse(fs.exists(foe));
|
||||
assertTrue(fs.exists(foe));
|
||||
assertTrue(hoopFs.delete(foe, true));
|
||||
assertFalse(fs.exists(foe));
|
||||
|
||||
hoopFs.close();
|
||||
fs.close();
|
||||
|
|
@ -333,19 +339,20 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
FileStatus status2 = fs.getFileStatus(new Path(path.toUri().getPath()));
|
||||
fs.close();
|
||||
|
||||
Assert.assertEquals(status2.getPermission(), status1.getPermission());
|
||||
Assert.assertEquals(status2.getPath().toUri().getPath(), status1.getPath().toUri().getPath());
|
||||
Assert.assertEquals(status2.getReplication(), status1.getReplication());
|
||||
Assert.assertEquals(status2.getBlockSize(), status1.getBlockSize());
|
||||
Assert.assertEquals(status2.getAccessTime(), status1.getAccessTime());
|
||||
Assert.assertEquals(status2.getModificationTime(), status1.getModificationTime());
|
||||
Assert.assertEquals(status2.getOwner(), status1.getOwner());
|
||||
Assert.assertEquals(status2.getGroup(), status1.getGroup());
|
||||
Assert.assertEquals(status2.getLen(), status1.getLen());
|
||||
assertEquals(status2.getPermission(), status1.getPermission());
|
||||
assertEquals(status2.getPath().toUri().getPath(),
|
||||
status1.getPath().toUri().getPath());
|
||||
assertEquals(status2.getReplication(), status1.getReplication());
|
||||
assertEquals(status2.getBlockSize(), status1.getBlockSize());
|
||||
assertEquals(status2.getAccessTime(), status1.getAccessTime());
|
||||
assertEquals(status2.getModificationTime(), status1.getModificationTime());
|
||||
assertEquals(status2.getOwner(), status1.getOwner());
|
||||
assertEquals(status2.getGroup(), status1.getGroup());
|
||||
assertEquals(status2.getLen(), status1.getLen());
|
||||
|
||||
FileStatus[] stati = fs.listStatus(path.getParent());
|
||||
Assert.assertEquals(stati.length, 1);
|
||||
Assert.assertEquals(stati[0].getPath().getName(), path.getName());
|
||||
assertEquals(stati.length, 1);
|
||||
assertEquals(stati[0].getPath().getName(), path.getName());
|
||||
}
|
||||
|
||||
private void testWorkingdirectory() throws Exception {
|
||||
|
|
@ -359,14 +366,15 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
}
|
||||
Path httpFSWorkingDir = fs.getWorkingDirectory();
|
||||
fs.close();
|
||||
Assert.assertEquals(httpFSWorkingDir.toUri().getPath(),
|
||||
assertEquals(httpFSWorkingDir.toUri().getPath(),
|
||||
workingDir.toUri().getPath());
|
||||
|
||||
fs = getHttpFSFileSystem();
|
||||
fs.setWorkingDirectory(new Path("/tmp"));
|
||||
workingDir = fs.getWorkingDirectory();
|
||||
fs.close();
|
||||
Assert.assertEquals(workingDir.toUri().getPath(), new Path("/tmp").toUri().getPath());
|
||||
assertEquals(workingDir.toUri().getPath(),
|
||||
new Path("/tmp").toUri().getPath());
|
||||
}
|
||||
|
||||
private void testMkdirs() throws Exception {
|
||||
|
|
@ -375,7 +383,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs.mkdirs(path);
|
||||
fs.close();
|
||||
fs = FileSystem.get(getProxiedFSConf());
|
||||
Assert.assertTrue(fs.exists(path));
|
||||
assertTrue(fs.exists(path));
|
||||
fs.close();
|
||||
}
|
||||
|
||||
|
|
@ -400,8 +408,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs.close();
|
||||
long atNew = status1.getAccessTime();
|
||||
long mtNew = status1.getModificationTime();
|
||||
Assert.assertEquals(mtNew, mt - 10);
|
||||
Assert.assertEquals(atNew, at - 20);
|
||||
assertEquals(mtNew, mt - 10);
|
||||
assertEquals(atNew, at - 20);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -419,7 +427,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
FileStatus status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
FsPermission permission2 = status1.getPermission();
|
||||
Assert.assertEquals(permission2, permission1);
|
||||
assertEquals(permission2, permission1);
|
||||
|
||||
//sticky bit
|
||||
fs = getHttpFSFileSystem();
|
||||
|
|
@ -431,8 +439,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
permission2 = status1.getPermission();
|
||||
Assert.assertTrue(permission2.getStickyBit());
|
||||
Assert.assertEquals(permission2, permission1);
|
||||
assertTrue(permission2.getStickyBit());
|
||||
assertEquals(permission2, permission1);
|
||||
}
|
||||
|
||||
private void testSetOwner() throws Exception {
|
||||
|
|
@ -454,8 +462,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs = FileSystem.get(getProxiedFSConf());
|
||||
FileStatus status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(status1.getOwner(), user);
|
||||
Assert.assertEquals(status1.getGroup(), group);
|
||||
assertEquals(status1.getOwner(), user);
|
||||
assertEquals(status1.getGroup(), group);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -475,7 +483,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs = FileSystem.get(getProxiedFSConf());
|
||||
FileStatus status1 = fs.getFileStatus(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(status1.getReplication(), (short) 1);
|
||||
assertEquals(status1.getReplication(), (short) 1);
|
||||
}
|
||||
|
||||
private void testChecksum() throws Exception {
|
||||
|
|
@ -491,9 +499,10 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs = getHttpFSFileSystem();
|
||||
FileChecksum httpChecksum = fs.getFileChecksum(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(httpChecksum.getAlgorithmName(), hdfsChecksum.getAlgorithmName());
|
||||
Assert.assertEquals(httpChecksum.getLength(), hdfsChecksum.getLength());
|
||||
Assert.assertArrayEquals(httpChecksum.getBytes(), hdfsChecksum.getBytes());
|
||||
assertEquals(httpChecksum.getAlgorithmName(),
|
||||
hdfsChecksum.getAlgorithmName());
|
||||
assertEquals(httpChecksum.getLength(), hdfsChecksum.getLength());
|
||||
assertArrayEquals(httpChecksum.getBytes(), hdfsChecksum.getBytes());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -508,12 +517,17 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs = getHttpFSFileSystem();
|
||||
ContentSummary httpContentSummary = fs.getContentSummary(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(httpContentSummary.getDirectoryCount(), hdfsContentSummary.getDirectoryCount());
|
||||
Assert.assertEquals(httpContentSummary.getFileCount(), hdfsContentSummary.getFileCount());
|
||||
Assert.assertEquals(httpContentSummary.getLength(), hdfsContentSummary.getLength());
|
||||
Assert.assertEquals(httpContentSummary.getQuota(), hdfsContentSummary.getQuota());
|
||||
Assert.assertEquals(httpContentSummary.getSpaceConsumed(), hdfsContentSummary.getSpaceConsumed());
|
||||
Assert.assertEquals(httpContentSummary.getSpaceQuota(), hdfsContentSummary.getSpaceQuota());
|
||||
assertEquals(httpContentSummary.getDirectoryCount(),
|
||||
hdfsContentSummary.getDirectoryCount());
|
||||
assertEquals(httpContentSummary.getFileCount(),
|
||||
hdfsContentSummary.getFileCount());
|
||||
assertEquals(httpContentSummary.getLength(),
|
||||
hdfsContentSummary.getLength());
|
||||
assertEquals(httpContentSummary.getQuota(), hdfsContentSummary.getQuota());
|
||||
assertEquals(httpContentSummary.getSpaceConsumed(),
|
||||
hdfsContentSummary.getSpaceConsumed());
|
||||
assertEquals(httpContentSummary.getSpaceQuota(),
|
||||
hdfsContentSummary.getSpaceQuota());
|
||||
}
|
||||
|
||||
/** Set xattr */
|
||||
|
|
@ -552,11 +566,11 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs = FileSystem.get(getProxiedFSConf());
|
||||
Map<String, byte[]> xAttrs = fs.getXAttrs(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(4, xAttrs.size());
|
||||
Assert.assertArrayEquals(value1, xAttrs.get(name1));
|
||||
Assert.assertArrayEquals(value2, xAttrs.get(name2));
|
||||
Assert.assertArrayEquals(new byte[0], xAttrs.get(name3));
|
||||
Assert.assertArrayEquals(value4, xAttrs.get(name4));
|
||||
assertEquals(4, xAttrs.size());
|
||||
assertArrayEquals(value1, xAttrs.get(name1));
|
||||
assertArrayEquals(value2, xAttrs.get(name2));
|
||||
assertArrayEquals(new byte[0], xAttrs.get(name3));
|
||||
assertArrayEquals(value4, xAttrs.get(name4));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -595,16 +609,16 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
names.add(name4);
|
||||
Map<String, byte[]> xAttrs = fs.getXAttrs(path, names);
|
||||
fs.close();
|
||||
Assert.assertEquals(4, xAttrs.size());
|
||||
Assert.assertArrayEquals(value1, xAttrs.get(name1));
|
||||
Assert.assertArrayEquals(value2, xAttrs.get(name2));
|
||||
Assert.assertArrayEquals(new byte[0], xAttrs.get(name3));
|
||||
Assert.assertArrayEquals(value4, xAttrs.get(name4));
|
||||
assertEquals(4, xAttrs.size());
|
||||
assertArrayEquals(value1, xAttrs.get(name1));
|
||||
assertArrayEquals(value2, xAttrs.get(name2));
|
||||
assertArrayEquals(new byte[0], xAttrs.get(name3));
|
||||
assertArrayEquals(value4, xAttrs.get(name4));
|
||||
|
||||
// Get specific xattr
|
||||
fs = getHttpFSFileSystem();
|
||||
byte[] value = fs.getXAttr(path, name1);
|
||||
Assert.assertArrayEquals(value1, value);
|
||||
assertArrayEquals(value1, value);
|
||||
final String name5 = "a1";
|
||||
try {
|
||||
value = fs.getXAttr(path, name5);
|
||||
|
|
@ -618,11 +632,11 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs = getHttpFSFileSystem();
|
||||
xAttrs = fs.getXAttrs(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(4, xAttrs.size());
|
||||
Assert.assertArrayEquals(value1, xAttrs.get(name1));
|
||||
Assert.assertArrayEquals(value2, xAttrs.get(name2));
|
||||
Assert.assertArrayEquals(new byte[0], xAttrs.get(name3));
|
||||
Assert.assertArrayEquals(value4, xAttrs.get(name4));
|
||||
assertEquals(4, xAttrs.size());
|
||||
assertArrayEquals(value1, xAttrs.get(name1));
|
||||
assertArrayEquals(value2, xAttrs.get(name2));
|
||||
assertArrayEquals(new byte[0], xAttrs.get(name3));
|
||||
assertArrayEquals(value4, xAttrs.get(name4));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -667,8 +681,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
fs = FileSystem.get(getProxiedFSConf());
|
||||
Map<String, byte[]> xAttrs = fs.getXAttrs(path);
|
||||
fs.close();
|
||||
Assert.assertEquals(1, xAttrs.size());
|
||||
Assert.assertArrayEquals(value2, xAttrs.get(name2));
|
||||
assertEquals(1, xAttrs.size());
|
||||
assertArrayEquals(value2, xAttrs.get(name2));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -700,11 +714,11 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
|
||||
fs = getHttpFSFileSystem();
|
||||
List<String> names = fs.listXAttrs(path);
|
||||
Assert.assertEquals(4, names.size());
|
||||
Assert.assertTrue(names.contains(name1));
|
||||
Assert.assertTrue(names.contains(name2));
|
||||
Assert.assertTrue(names.contains(name3));
|
||||
Assert.assertTrue(names.contains(name4));
|
||||
assertEquals(4, names.size());
|
||||
assertTrue(names.contains(name1));
|
||||
assertTrue(names.contains(name2));
|
||||
assertTrue(names.contains(name3));
|
||||
assertTrue(names.contains(name4));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -715,18 +729,26 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
* @throws Exception
|
||||
*/
|
||||
private void assertSameAcls(AclStatus a, AclStatus b) throws Exception {
|
||||
Assert.assertTrue(a.getOwner().equals(b.getOwner()));
|
||||
Assert.assertTrue(a.getGroup().equals(b.getGroup()));
|
||||
Assert.assertTrue(a.isStickyBit() == b.isStickyBit());
|
||||
Assert.assertTrue(a.getEntries().size() == b.getEntries().size());
|
||||
assertTrue(a.getOwner().equals(b.getOwner()));
|
||||
assertTrue(a.getGroup().equals(b.getGroup()));
|
||||
assertTrue(a.isStickyBit() == b.isStickyBit());
|
||||
assertTrue(a.getEntries().size() == b.getEntries().size());
|
||||
for (AclEntry e : a.getEntries()) {
|
||||
Assert.assertTrue(b.getEntries().contains(e));
|
||||
assertTrue(b.getEntries().contains(e));
|
||||
}
|
||||
for (AclEntry e : b.getEntries()) {
|
||||
Assert.assertTrue(a.getEntries().contains(e));
|
||||
assertTrue(a.getEntries().contains(e));
|
||||
}
|
||||
}
|
||||
|
||||
private static void assertSameAclBit(FileSystem expected, FileSystem actual,
|
||||
Path path) throws IOException {
|
||||
FileStatus expectedFileStatus = expected.getFileStatus(path);
|
||||
FileStatus actualFileStatus = actual.getFileStatus(path);
|
||||
assertEquals(actualFileStatus.getPermission().getAclBit(),
|
||||
expectedFileStatus.getPermission().getAclBit());
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple ACL tests on a file: Set an acl, add an acl, remove one acl,
|
||||
* and remove all acls.
|
||||
|
|
@ -755,26 +777,31 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
AclStatus proxyAclStat = proxyFs.getAclStatus(path);
|
||||
AclStatus httpfsAclStat = httpfs.getAclStatus(path);
|
||||
assertSameAcls(httpfsAclStat, proxyAclStat);
|
||||
assertSameAclBit(httpfs, proxyFs, path);
|
||||
|
||||
httpfs.setAcl(path, AclEntry.parseAclSpec(aclSet,true));
|
||||
proxyAclStat = proxyFs.getAclStatus(path);
|
||||
httpfsAclStat = httpfs.getAclStatus(path);
|
||||
assertSameAcls(httpfsAclStat, proxyAclStat);
|
||||
assertSameAclBit(httpfs, proxyFs, path);
|
||||
|
||||
httpfs.modifyAclEntries(path, AclEntry.parseAclSpec(aclUser2, true));
|
||||
proxyAclStat = proxyFs.getAclStatus(path);
|
||||
httpfsAclStat = httpfs.getAclStatus(path);
|
||||
assertSameAcls(httpfsAclStat, proxyAclStat);
|
||||
assertSameAclBit(httpfs, proxyFs, path);
|
||||
|
||||
httpfs.removeAclEntries(path, AclEntry.parseAclSpec(rmAclUser1, false));
|
||||
proxyAclStat = proxyFs.getAclStatus(path);
|
||||
httpfsAclStat = httpfs.getAclStatus(path);
|
||||
assertSameAcls(httpfsAclStat, proxyAclStat);
|
||||
assertSameAclBit(httpfs, proxyFs, path);
|
||||
|
||||
httpfs.removeAcl(path);
|
||||
proxyAclStat = proxyFs.getAclStatus(path);
|
||||
httpfsAclStat = httpfs.getAclStatus(path);
|
||||
assertSameAcls(httpfsAclStat, proxyAclStat);
|
||||
assertSameAclBit(httpfs, proxyFs, path);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -797,25 +824,46 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
AclStatus proxyAclStat = proxyFs.getAclStatus(dir);
|
||||
AclStatus httpfsAclStat = httpfs.getAclStatus(dir);
|
||||
assertSameAcls(httpfsAclStat, proxyAclStat);
|
||||
assertSameAclBit(httpfs, proxyFs, dir);
|
||||
|
||||
/* Set a default ACL on the directory */
|
||||
httpfs.setAcl(dir, (AclEntry.parseAclSpec(defUser1,true)));
|
||||
proxyAclStat = proxyFs.getAclStatus(dir);
|
||||
httpfsAclStat = httpfs.getAclStatus(dir);
|
||||
assertSameAcls(httpfsAclStat, proxyAclStat);
|
||||
assertSameAclBit(httpfs, proxyFs, dir);
|
||||
|
||||
/* Remove the default ACL */
|
||||
httpfs.removeDefaultAcl(dir);
|
||||
proxyAclStat = proxyFs.getAclStatus(dir);
|
||||
httpfsAclStat = httpfs.getAclStatus(dir);
|
||||
assertSameAcls(httpfsAclStat, proxyAclStat);
|
||||
assertSameAclBit(httpfs, proxyFs, dir);
|
||||
}
|
||||
|
||||
private void testEncryption() throws Exception {
|
||||
if (isLocalFS()) {
|
||||
return;
|
||||
}
|
||||
FileSystem proxyFs = FileSystem.get(getProxiedFSConf());
|
||||
FileSystem httpFs = getHttpFSFileSystem();
|
||||
FileStatus proxyStatus = proxyFs.getFileStatus(TestHdfsHelper
|
||||
.ENCRYPTED_FILE);
|
||||
assertTrue(proxyStatus.isEncrypted());
|
||||
FileStatus httpStatus = httpFs.getFileStatus(TestHdfsHelper
|
||||
.ENCRYPTED_FILE);
|
||||
assertTrue(httpStatus.isEncrypted());
|
||||
proxyStatus = proxyFs.getFileStatus(new Path("/"));
|
||||
httpStatus = httpFs.getFileStatus(new Path("/"));
|
||||
assertFalse(proxyStatus.isEncrypted());
|
||||
assertFalse(httpStatus.isEncrypted());
|
||||
}
|
||||
|
||||
protected enum Operation {
|
||||
GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS,
|
||||
WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER,
|
||||
SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR,
|
||||
GET_XATTRS, REMOVE_XATTR, LIST_XATTRS
|
||||
GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION
|
||||
}
|
||||
|
||||
private void operation(Operation op) throws Exception {
|
||||
|
|
@ -889,6 +937,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
|
|||
case LIST_XATTRS:
|
||||
testListXAttrs();
|
||||
break;
|
||||
case ENCRYPTION:
|
||||
testEncryption();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,10 +21,14 @@ import java.io.File;
|
|||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.FileSystemTestHelper;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.junit.Test;
|
||||
import org.junit.runners.model.FrameworkMethod;
|
||||
|
|
@ -129,6 +133,9 @@ public class TestHdfsHelper extends TestDirHelper {
|
|||
return new Configuration(conf);
|
||||
}
|
||||
|
||||
public static final Path ENCRYPTION_ZONE = new Path("/ez");
|
||||
public static final Path ENCRYPTED_FILE = new Path("/ez/encfile");
|
||||
|
||||
private static MiniDFSCluster MINI_DFS = null;
|
||||
|
||||
private static synchronized MiniDFSCluster startMiniHdfs(Configuration conf) throws Exception {
|
||||
|
|
@ -148,14 +155,28 @@ public class TestHdfsHelper extends TestDirHelper {
|
|||
conf.set("hadoop.security.authentication", "simple");
|
||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
|
||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
|
||||
FileSystemTestHelper helper = new FileSystemTestHelper();
|
||||
final String jceksPath = JavaKeyStoreProvider.SCHEME_NAME + "://file" +
|
||||
new Path(helper.getTestRootDir(), "test.jks").toUri();
|
||||
conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, jceksPath);
|
||||
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
|
||||
builder.numDataNodes(2);
|
||||
MiniDFSCluster miniHdfs = builder.build();
|
||||
FileSystem fileSystem = miniHdfs.getFileSystem();
|
||||
final String testkey = "testkey";
|
||||
DFSTestUtil.createKey(testkey, miniHdfs, conf);
|
||||
|
||||
DistributedFileSystem fileSystem = miniHdfs.getFileSystem();
|
||||
fileSystem.getClient().setKeyProvider(miniHdfs.getNameNode()
|
||||
.getNamesystem().getProvider());
|
||||
|
||||
fileSystem.mkdirs(new Path("/tmp"));
|
||||
fileSystem.mkdirs(new Path("/user"));
|
||||
fileSystem.setPermission(new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx"));
|
||||
fileSystem.setPermission(new Path("/user"), FsPermission.valueOf("-rwxrwxrwx"));
|
||||
fileSystem.mkdirs(ENCRYPTION_ZONE);
|
||||
fileSystem.createEncryptionZone(ENCRYPTION_ZONE, testkey);
|
||||
fileSystem.create(ENCRYPTED_FILE).close();
|
||||
|
||||
MINI_DFS = miniHdfs;
|
||||
}
|
||||
return MINI_DFS;
|
||||
|
|
|
|||
Loading…
Reference in New Issue