HDFS-6509 addendum, extra file
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/fs-encryption@1614491 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Wang
parent
407bb3d3e4
commit
3a90228c30
|
|
@ -0,0 +1,348 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hdfs;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
|
||||
import org.apache.hadoop.crypto.key.KeyProviderFactory;
|
||||
import org.apache.hadoop.fs.FileContext;
|
||||
import org.apache.hadoop.fs.FileContextTestWrapper;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystemTestHelper;
|
||||
import org.apache.hadoop.fs.FileSystemTestWrapper;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.hdfs.client.HdfsAdmin;
|
||||
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesEqual;
|
||||
import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesNotEqual;
|
||||
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
|
||||
import static org.apache.hadoop.test.GenericTestUtils.assertMatches;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
public class TestReservedRawPaths {
|
||||
|
||||
private Configuration conf;
|
||||
private FileSystemTestHelper fsHelper;
|
||||
|
||||
private MiniDFSCluster cluster;
|
||||
private HdfsAdmin dfsAdmin;
|
||||
private DistributedFileSystem fs;
|
||||
|
||||
protected FileSystemTestWrapper fsWrapper;
|
||||
protected FileContextTestWrapper fcWrapper;
|
||||
|
||||
@Before
|
||||
public void setup() throws IOException {
|
||||
conf = new HdfsConfiguration();
|
||||
fsHelper = new FileSystemTestHelper();
|
||||
// Set up java key store
|
||||
String testRoot = fsHelper.getTestRootDir();
|
||||
File testRootDir = new File(testRoot).getAbsoluteFile();
|
||||
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
|
||||
JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir + "/test.jks"
|
||||
);
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
|
||||
Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE);
|
||||
fs = cluster.getFileSystem();
|
||||
fsWrapper = new FileSystemTestWrapper(cluster.getFileSystem());
|
||||
fcWrapper = new FileContextTestWrapper(
|
||||
FileContext.getFileContext(cluster.getURI(), conf));
|
||||
dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
|
||||
// Need to set the client's KeyProvider to the NN's for JKS,
|
||||
// else the updates do not get flushed properly
|
||||
fs.getClient().provider = cluster.getNameNode().getNamesystem()
|
||||
.getProvider();
|
||||
}
|
||||
|
||||
@After
|
||||
public void teardown() {
|
||||
if (cluster != null) {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic read/write tests of raw files.
|
||||
* Create a non-encrypted file
|
||||
* Create an encryption zone
|
||||
* Verify that non-encrypted file contents and decrypted file in EZ are equal
|
||||
* Compare the raw encrypted bytes of the file with the decrypted version to
|
||||
* ensure they're different
|
||||
* Compare the raw and non-raw versions of the non-encrypted file to ensure
|
||||
* they're the same.
|
||||
*/
|
||||
@Test(timeout = 120000)
|
||||
public void testReadWriteRaw() throws Exception {
|
||||
// Create a base file for comparison
|
||||
final Path baseFile = new Path("/base");
|
||||
final int len = 8192;
|
||||
DFSTestUtil.createFile(fs, baseFile, len, (short) 1, 0xFEED);
|
||||
// Create the first enc file
|
||||
final Path zone = new Path("/zone");
|
||||
fs.mkdirs(zone);
|
||||
dfsAdmin.createEncryptionZone(zone, null);
|
||||
final Path encFile1 = new Path(zone, "myfile");
|
||||
DFSTestUtil.createFile(fs, encFile1, len, (short) 1, 0xFEED);
|
||||
// Read them back in and compare byte-by-byte
|
||||
verifyFilesEqual(fs, baseFile, encFile1, len);
|
||||
// Raw file should be different from encrypted file
|
||||
final Path encFile1Raw = new Path(zone, "/.reserved/raw/zone/myfile");
|
||||
verifyFilesNotEqual(fs, encFile1Raw, encFile1, len);
|
||||
// Raw file should be same as /base which is not in an EZ
|
||||
final Path baseFileRaw = new Path(zone, "/.reserved/raw/base");
|
||||
verifyFilesEqual(fs, baseFile, baseFileRaw, len);
|
||||
}
|
||||
|
||||
private void assertPathEquals(Path p1, Path p2) throws IOException {
|
||||
final FileStatus p1Stat = fs.getFileStatus(p1);
|
||||
final FileStatus p2Stat = fs.getFileStatus(p2);
|
||||
|
||||
/*
|
||||
* Use accessTime and modificationTime as substitutes for INode to check
|
||||
* for resolution to the same underlying file.
|
||||
*/
|
||||
assertEquals("Access times not equal", p1Stat.getAccessTime(),
|
||||
p2Stat.getAccessTime());
|
||||
assertEquals("Modification times not equal", p1Stat.getModificationTime(),
|
||||
p2Stat.getModificationTime());
|
||||
assertEquals("pathname1 not equal", p1,
|
||||
Path.getPathWithoutSchemeAndAuthority(p1Stat.getPath()));
|
||||
assertEquals("pathname1 not equal", p2,
|
||||
Path.getPathWithoutSchemeAndAuthority(p2Stat.getPath()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests that getFileStatus on raw and non raw resolve to the same
|
||||
* file.
|
||||
*/
|
||||
@Test(timeout = 120000)
|
||||
public void testGetFileStatus() throws Exception {
|
||||
final Path zone = new Path("zone");
|
||||
final Path slashZone = new Path("/", zone);
|
||||
fs.mkdirs(slashZone);
|
||||
dfsAdmin.createEncryptionZone(slashZone, null);
|
||||
|
||||
final Path base = new Path("base");
|
||||
final Path reservedRaw = new Path("/.reserved/raw");
|
||||
final Path baseRaw = new Path(reservedRaw, base);
|
||||
final int len = 8192;
|
||||
DFSTestUtil.createFile(fs, baseRaw, len, (short) 1, 0xFEED);
|
||||
assertPathEquals(new Path("/", base), baseRaw);
|
||||
|
||||
/* Repeat the test for a file in an ez. */
|
||||
final Path ezEncFile = new Path(slashZone, base);
|
||||
final Path ezRawEncFile =
|
||||
new Path(new Path(reservedRaw, zone), base);
|
||||
DFSTestUtil.createFile(fs, ezEncFile, len, (short) 1, 0xFEED);
|
||||
assertPathEquals(ezEncFile, ezRawEncFile);
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testReservedRoot() throws Exception {
|
||||
final Path root = new Path("/");
|
||||
final Path rawRoot = new Path("/.reserved/raw");
|
||||
final Path rawRootSlash = new Path("/.reserved/raw/");
|
||||
assertPathEquals(root, rawRoot);
|
||||
assertPathEquals(root, rawRootSlash);
|
||||
}
|
||||
|
||||
/* Verify mkdir works ok in .reserved/raw directory. */
|
||||
@Test(timeout = 120000)
|
||||
public void testReservedRawMkdir() throws Exception {
|
||||
final Path zone = new Path("zone");
|
||||
final Path slashZone = new Path("/", zone);
|
||||
fs.mkdirs(slashZone);
|
||||
dfsAdmin.createEncryptionZone(slashZone, null);
|
||||
final Path rawRoot = new Path("/.reserved/raw");
|
||||
final Path dir1 = new Path("dir1");
|
||||
final Path rawDir1 = new Path(rawRoot, dir1);
|
||||
fs.mkdirs(rawDir1);
|
||||
assertPathEquals(rawDir1, new Path("/", dir1));
|
||||
fs.delete(rawDir1, true);
|
||||
final Path rawZone = new Path(rawRoot, zone);
|
||||
final Path rawDir1EZ = new Path(rawZone, dir1);
|
||||
fs.mkdirs(rawDir1EZ);
|
||||
assertPathEquals(rawDir1EZ, new Path(slashZone, dir1));
|
||||
fs.delete(rawDir1EZ, true);
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testRelativePathnames() throws Exception {
|
||||
final Path baseFileRaw = new Path("/.reserved/raw/base");
|
||||
final int len = 8192;
|
||||
DFSTestUtil.createFile(fs, baseFileRaw, len, (short) 1, 0xFEED);
|
||||
|
||||
final Path root = new Path("/");
|
||||
final Path rawRoot = new Path("/.reserved/raw");
|
||||
assertPathEquals(root, new Path(rawRoot, "../raw"));
|
||||
assertPathEquals(root, new Path(rawRoot, "../../.reserved/raw"));
|
||||
assertPathEquals(baseFileRaw, new Path(rawRoot, "../raw/base"));
|
||||
assertPathEquals(baseFileRaw, new Path(rawRoot,
|
||||
"../../.reserved/raw/base"));
|
||||
assertPathEquals(baseFileRaw, new Path(rawRoot,
|
||||
"../../.reserved/raw/base/../base"));
|
||||
assertPathEquals(baseFileRaw, new Path(
|
||||
"/.reserved/../.reserved/raw/../raw/base"));
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testAdminAccessOnly() throws Exception {
|
||||
final Path zone = new Path("zone");
|
||||
final Path slashZone = new Path("/", zone);
|
||||
fs.mkdirs(slashZone);
|
||||
dfsAdmin.createEncryptionZone(slashZone, null);
|
||||
final Path base = new Path("base");
|
||||
final Path reservedRaw = new Path("/.reserved/raw");
|
||||
final int len = 8192;
|
||||
|
||||
/* Test failure of create file in reserved/raw as non admin */
|
||||
final UserGroupInformation user = UserGroupInformation.
|
||||
createUserForTesting("user", new String[] { "mygroup" });
|
||||
user.doAs(new PrivilegedExceptionAction<Object>() {
|
||||
@Override
|
||||
public Object run() throws Exception {
|
||||
final DistributedFileSystem fs = cluster.getFileSystem();
|
||||
try {
|
||||
final Path ezRawEncFile =
|
||||
new Path(new Path(reservedRaw, zone), base);
|
||||
DFSTestUtil.createFile(fs, ezRawEncFile, len, (short) 1, 0xFEED);
|
||||
fail("access to /.reserved/raw is superuser-only operation");
|
||||
} catch (AccessControlException e) {
|
||||
assertExceptionContains("Superuser privilege is required", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
/* Test failure of getFileStatus in reserved/raw as non admin */
|
||||
final Path ezRawEncFile = new Path(new Path(reservedRaw, zone), base);
|
||||
DFSTestUtil.createFile(fs, ezRawEncFile, len, (short) 1, 0xFEED);
|
||||
user.doAs(new PrivilegedExceptionAction<Object>() {
|
||||
@Override
|
||||
public Object run() throws Exception {
|
||||
final DistributedFileSystem fs = cluster.getFileSystem();
|
||||
try {
|
||||
fs.getFileStatus(ezRawEncFile);
|
||||
fail("access to /.reserved/raw is superuser-only operation");
|
||||
} catch (AccessControlException e) {
|
||||
assertExceptionContains("Superuser privilege is required", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
/* Test failure of listStatus in reserved/raw as non admin */
|
||||
user.doAs(new PrivilegedExceptionAction<Object>() {
|
||||
@Override
|
||||
public Object run() throws Exception {
|
||||
final DistributedFileSystem fs = cluster.getFileSystem();
|
||||
try {
|
||||
fs.listStatus(ezRawEncFile);
|
||||
fail("access to /.reserved/raw is superuser-only operation");
|
||||
} catch (AccessControlException e) {
|
||||
assertExceptionContains("Superuser privilege is required", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
fs.setPermission(new Path("/"), new FsPermission((short) 0777));
|
||||
/* Test failure of mkdir in reserved/raw as non admin */
|
||||
user.doAs(new PrivilegedExceptionAction<Object>() {
|
||||
@Override
|
||||
public Object run() throws Exception {
|
||||
final DistributedFileSystem fs = cluster.getFileSystem();
|
||||
final Path d1 = new Path(reservedRaw, "dir1");
|
||||
try {
|
||||
fs.mkdirs(d1);
|
||||
fail("access to /.reserved/raw is superuser-only operation");
|
||||
} catch (AccessControlException e) {
|
||||
assertExceptionContains("Superuser privilege is required", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testListDotReserved() throws Exception {
|
||||
// Create a base file for comparison
|
||||
final Path baseFileRaw = new Path("/.reserved/raw/base");
|
||||
final int len = 8192;
|
||||
DFSTestUtil.createFile(fs, baseFileRaw, len, (short) 1, 0xFEED);
|
||||
|
||||
/*
|
||||
* Ensure that you can't list /.reserved. Ever.
|
||||
*/
|
||||
try {
|
||||
fs.listStatus(new Path("/.reserved"));
|
||||
fail("expected FNFE");
|
||||
} catch (FileNotFoundException e) {
|
||||
assertExceptionContains("/.reserved does not exist", e);
|
||||
}
|
||||
|
||||
try {
|
||||
fs.listStatus(new Path("/.reserved/.inodes"));
|
||||
fail("expected FNFE");
|
||||
} catch (FileNotFoundException e) {
|
||||
assertExceptionContains(
|
||||
"/.reserved/.inodes does not exist", e);
|
||||
}
|
||||
|
||||
final FileStatus[] fileStatuses = fs.listStatus(new Path("/.reserved/raw"));
|
||||
assertEquals("expected 1 entry", fileStatuses.length, 1);
|
||||
assertMatches(fileStatuses[0].getPath().toString(), "/.reserved/raw/base");
|
||||
}
|
||||
|
||||
@Test(timeout = 120000)
|
||||
public void testListRecursive() throws Exception {
|
||||
Path rootPath = new Path("/");
|
||||
Path p = rootPath;
|
||||
for (int i = 0; i < 3; i++) {
|
||||
p = new Path(p, "dir" + i);
|
||||
fs.mkdirs(p);
|
||||
}
|
||||
|
||||
Path curPath = new Path("/.reserved/raw");
|
||||
int cnt = 0;
|
||||
FileStatus[] fileStatuses = fs.listStatus(curPath);
|
||||
while (fileStatuses != null && fileStatuses.length > 0) {
|
||||
FileStatus f = fileStatuses[0];
|
||||
assertMatches(f.getPath().toString(), "/.reserved/raw");
|
||||
curPath = Path.getPathWithoutSchemeAndAuthority(f.getPath());
|
||||
cnt++;
|
||||
fileStatuses = fs.listStatus(curPath);
|
||||
}
|
||||
assertEquals(3, cnt);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue