From 4bd2f58a83e99b0ce9007668db149d6b075a468e Mon Sep 17 00:00:00 2001 From: Xing Lin Date: Thu, 31 Mar 2022 13:58:29 -0700 Subject: [PATCH] HADOOP-18144: getTrashRoot in ViewFileSystem should return a path in ViewFS (#4124) * HADOOP-18144: getTrashRoot in ViewFileSystem should return a path in ViewFS. To get the new behavior, define fs.viewfs.trash.force-inside-mount-point to be true. If the trash root for path p is in the same mount point as path p, and one of: * The mount point isn't at the top of the target fs. * The resolved path of path is root (eg it is the fallback FS). * The trash root isn't in user's target fs home directory. get the corresponding viewFS path for the trash root and return it. Otherwise, use /.Trash/. Signed-off-by: Owen O'Malley (cherry picked from commit 8b8158f02df18424b2406fd66b34c1bdf3d7ab55) --- .../apache/hadoop/fs/TrashPolicyDefault.java | 4 +- .../apache/hadoop/fs/viewfs/Constants.java | 7 +- .../apache/hadoop/fs/viewfs/InodeTree.java | 2 +- .../hadoop/fs/viewfs/ViewFileSystem.java | 195 +++++++++++------- .../TestViewFileSystemLocalFileSystem.java | 9 + ...ileSystemWithAuthorityLocalFileSystem.java | 10 + .../fs/viewfs/ViewFileSystemBaseTest.java | 107 ++++++---- .../fs/viewfs/TestViewFileSystemHdfs.java | 8 + 8 files changed, 213 insertions(+), 129 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index 265e967b01e..4ac8f268fad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -171,8 +171,8 @@ public class TrashPolicyDefault extends TrashPolicy { cause = e; } } - throw (IOException) - new IOException("Failed to move to trash: " + path).initCause(cause); + throw new IOException("Failed to move " + path + " to trash " + trashPath, + cause); } @SuppressWarnings("deprecation") diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index 7a0a6661f1e..7088ef307fa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -87,8 +87,9 @@ public interface Constants { boolean CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT = true; /** - * Enable ViewFileSystem to return a trashRoot which is local to mount point. + * Force ViewFileSystem to return a trashRoot that is inside a mount point. */ - String CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH = "fs.viewfs.mount.point.local.trash"; - boolean CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT = false; + String CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT = + "fs.viewfs.trash.force-inside-mount-point"; + boolean CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT = false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index ecd89e3416c..4b73ddb58df 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -410,7 +410,7 @@ abstract class InodeTree { return rootFallbackLink != null; } - private INodeLink getRootFallbackLink() { + protected INodeLink getRootFallbackLink() { Preconditions.checkState(root.isInternalDir()); return rootFallbackLink; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 25b82e52199..709c7042b2f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -20,8 +20,8 @@ package org.apache.hadoop.fs.viewfs; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT; -import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH; -import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT; import com.google.common.base.Function; import java.io.FileNotFoundException; @@ -29,7 +29,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -945,47 +944,77 @@ public class ViewFileSystem extends FileSystem { * Get the trash root directory for current user when the path * specified is deleted. * - * If CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is not set, return - * the default trash root from targetFS. + * If FORCE_INSIDE_MOUNT_POINT flag is not set, return the default trash root + * from targetFS. * - * When CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is set to true, - * 1) If path p is in fallback FS or from the same mount point as the default - * trash root for targetFS, return the default trash root for targetFS. - * 2) else, return a trash root in the mounted targetFS - * (/mntpoint/.Trash/{user}) + * When FORCE_INSIDE_MOUNT_POINT is set to true, + *
    + *
  1. + * If the trash root for path p is in the same mount point as path p, + * and one of: + *
      + *
    1. The mount point isn't at the top of the target fs.
      2. + *
    2. The resolved path of path is root (in fallback FS).
      3. + *
    3. The trash isn't in user's target fs home directory + * get the corresponding viewFS path for the trash root and return + * it. + *
      4. + *
    + *
    2. + *
  2. + * else, return the trash root under the root of the mount point + * (/{mntpoint}/.Trash/{user}). + *
    3. + *
+ * + * These conditions handle several different important cases: + *
    + *
  • File systems may need to have more local trash roots, such as + * encryption zones or snapshot roots.
    • + *
  • The fallback mount should use the user's home directory.
    • + *
  • Cloud storage systems should not use trash in an implicity defined + * home directory, per a container, unless it is the fallback fs.
    • + *
* * @param path the trash root of the path to be determined. * @return the trash root path. */ @Override public Path getTrashRoot(Path path) { - boolean useMountPointLocalTrash = - config.getBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, - CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT); try { InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), true); + Path targetFSTrashRoot = + res.targetFileSystem.getTrashRoot(res.remainingPath); - Path trashRoot = res.targetFileSystem.getTrashRoot(res.remainingPath); - if (!useMountPointLocalTrash) { - return trashRoot; + // Allow clients to use old behavior of delegating to target fs. + if (!config.getBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, + CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT)) { + return targetFSTrashRoot; + } + + // The trash root path from the target fs + String targetFSTrashRootPath = targetFSTrashRoot.toUri().getPath(); + // The mount point path in the target fs + String mountTargetPath = res.targetFileSystem.getUri().getPath(); + if (!mountTargetPath.endsWith("/")) { + mountTargetPath = mountTargetPath + "/"; + } + + Path targetFsUserHome = res.targetFileSystem.getHomeDirectory(); + if (targetFSTrashRootPath.startsWith(mountTargetPath) && + !(mountTargetPath.equals(ROOT_PATH.toString()) && + !res.resolvedPath.equals(ROOT_PATH.toString()) && + (targetFsUserHome != null && targetFSTrashRootPath.startsWith( + targetFsUserHome.toUri().getPath())))) { + String relativeTrashRoot = + targetFSTrashRootPath.substring(mountTargetPath.length()); + return makeQualified(new Path(res.resolvedPath, relativeTrashRoot)); } else { - // Path p is either in a mount point or in the fallback FS - - if (ROOT_PATH.equals(new Path(res.resolvedPath)) - || trashRoot.toUri().getPath().startsWith(res.resolvedPath)) { - // Path p is in the fallback FS or targetFileSystem.trashRoot is in - // the same mount point as Path p - return trashRoot; - } else { - // targetFileSystem.trashRoot is in a different mount point from - // Path p. Return the trash root for the mount point. - Path mountPointRoot = - res.targetFileSystem.getFileStatus(new Path("/")).getPath(); - return new Path(mountPointRoot, - TRASH_PREFIX + "/" + ugi.getShortUserName()); - } + // Return the trash root for the mount point. + return makeQualified(new Path(res.resolvedPath, + TRASH_PREFIX + "/" + ugi.getShortUserName())); } } catch (IOException | IllegalArgumentException e) { throw new NotInMountpointException(path, "getTrashRoot"); @@ -995,72 +1024,78 @@ public class ViewFileSystem extends FileSystem { /** * Get all the trash roots for current user or all users. * + * When FORCE_INSIDE_MOUNT_POINT is set to true, we also return trash roots + * under the root of each mount point, with their viewFS paths. + * * @param allUsers return trash roots for all users if true. * @return all Trash root directories. */ @Override public Collection getTrashRoots(boolean allUsers) { - List trashRoots = new ArrayList<>(); + // A map from targetFSPath -> FileStatus. + // FileStatus can be from targetFS or viewFS. + HashMap trashRoots = new HashMap<>(); for (FileSystem fs : getChildFileSystems()) { - trashRoots.addAll(fs.getTrashRoots(allUsers)); + for (FileStatus trash : fs.getTrashRoots(allUsers)) { + trashRoots.put(trash.getPath(), trash); + } } - // Add trash dirs for each mount point - boolean useMountPointLocalTrash = - config.getBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, - CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT); - if (useMountPointLocalTrash) { + // Return trashRoots if FORCE_INSIDE_MOUNT_POINT is disabled. + if (!config.getBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, + CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT)) { + return trashRoots.values(); + } - Set currentTrashPaths = new HashSet<>(); - for (FileStatus file : trashRoots) { - currentTrashPaths.add(file.getPath()); - } + // Get trash roots in TRASH_PREFIX dir inside mount points and fallback FS. + List> mountPoints = + fsState.getMountPoints(); + // If we have a fallback FS, add a mount point for it as <"", fallback FS>. + // The source path of a mount point shall not end with '/', thus for + // fallback fs, we set its mount point src as "". + if (fsState.getRootFallbackLink() != null) { + mountPoints.add(new InodeTree.MountPoint<>("", + fsState.getRootFallbackLink())); + } - MountPoint[] mountPoints = getMountPoints(); - try { - for (int i = 0; i < mountPoints.length; i++) { - Path trashRoot = makeQualified( - new Path(mountPoints[i].getSrc() + "/" + TRASH_PREFIX)); + try { + for (InodeTree.MountPoint mountPoint : mountPoints) { - // Continue if trashRoot does not exist for this filesystem - if (!exists(trashRoot)) { - continue; + Path trashRoot = + makeQualified(new Path(mountPoint.src + "/" + TRASH_PREFIX)); + + // Continue if trashRoot does not exist for this mount point + if (!exists(trashRoot)) { + continue; + } + + FileSystem targetFS = mountPoint.target.getTargetFileSystem(); + if (!allUsers) { + Path userTrashRoot = new Path(trashRoot, ugi.getShortUserName()); + if (exists(userTrashRoot)) { + Path targetFSUserTrashRoot = targetFS.makeQualified( + new Path(targetFS.getUri().getPath(), + TRASH_PREFIX + "/" + ugi.getShortUserName())); + trashRoots.put(targetFSUserTrashRoot, getFileStatus(userTrashRoot)); } - - InodeTree.ResolveResult res = - fsState.resolve(getUriPath(trashRoot), true); - - if (!allUsers) { - Path userTrash = - new Path("/" + TRASH_PREFIX + "/" + ugi.getShortUserName()); - try { - FileStatus file = res.targetFileSystem.getFileStatus(userTrash); - if (!currentTrashPaths.contains(file.getPath())) { - trashRoots.add(file); - currentTrashPaths.add(file.getPath()); - } - } catch (FileNotFoundException ignored) { - } - } else { - FileStatus[] targetFsTrashRoots = - res.targetFileSystem.listStatus(new Path("/" + TRASH_PREFIX)); - for (FileStatus file : targetFsTrashRoots) { - // skip if we already include it in currentTrashPaths - if (currentTrashPaths.contains(file.getPath())) { - continue; - } - - trashRoots.add(file); - currentTrashPaths.add(file.getPath()); - } + } else { + FileStatus[] mountPointTrashRoots = listStatus(trashRoot); + for (FileStatus trash : mountPointTrashRoots) { + // Remove the mountPoint and the leading '/' to get the + // relative targetFsTrash path + String targetFsTrash = trash.getPath().toUri().getPath() + .substring(mountPoint.src.length() + 1); + Path targetFsTrashPath = targetFS.makeQualified( + new Path(targetFS.getUri().getPath(), targetFsTrash)); + trashRoots.put(targetFsTrashPath, trash); } } - } catch (IOException e) { - LOG.warn("Exception in get all trash roots", e); } + } catch (IOException e) { + LOG.warn("Exception in get all trash roots for mount points", e); } - return trashRoots; + return trashRoots.values(); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java index 808d8b06c35..adc5db87e77 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java @@ -33,6 +33,8 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.Before; @@ -61,6 +63,13 @@ public class TestViewFileSystemLocalFileSystem extends ViewFileSystemBaseTest { } + @Override + Path getTrashRootInFallBackFS() throws IOException { + return new Path( + "/" + TRASH_PREFIX + "/" + UserGroupInformation.getCurrentUser() + .getShortUserName()); + } + @Test public void testNflyWriteSimple() throws IOException { LOG.info("Starting testNflyWriteSimple"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java index 877c2228c1e..9223338f34b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java @@ -25,6 +25,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; +import org.apache.hadoop.security.UserGroupInformation; +import java.io.IOException; import org.junit.After; import org.junit.Assert; @@ -63,6 +66,13 @@ public class TestViewFileSystemWithAuthorityLocalFileSystem extends ViewFileSyst super.tearDown(); } + @Override + Path getTrashRootInFallBackFS() throws IOException { + return new Path( + "/" + TRASH_PREFIX + "/" + UserGroupInformation.getCurrentUser() + .getShortUserName()); + } + @Override @Test public void testBasicPaths() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 3526f2534bb..f1ec91e4c16 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -38,7 +38,7 @@ import static org.apache.hadoop.fs.FileSystemTestHelper.*; import org.apache.hadoop.fs.permission.AclEntry; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; -import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT; import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import static org.apache.hadoop.test.GenericTestUtils.*; import static org.junit.Assert.*; @@ -1022,62 +1022,62 @@ abstract public class ViewFileSystemBaseTest { Assert.assertTrue("", fsView.getTrashRoots(true).size() > 0); } + // Default implementation of getTrashRoot for a fallback FS mounted at root: + // e.g., fallbackFS.uri.getPath = '/' + Path getTrashRootInFallBackFS() throws IOException { + return new Path(fsTarget.getHomeDirectory().toUri().getPath(), + TRASH_PREFIX); + } + /** - * Test the localized trash root for getTrashRoot. + * Test TRASH_FORCE_INSIDE_MOUNT_POINT feature for getTrashRoot. */ @Test - public void testTrashRootLocalizedTrash() throws IOException { + public void testTrashRootForceInsideMountPoint() throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); Configuration conf2 = new Configuration(conf); - conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true); + ConfigUtil.addLinkFallback(conf2, targetTestRoot.toUri()); FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); - // Case 1: path p not in the default FS. - // Return a trash root within the mount point. + // Case 1: path p in the /data mount point. + // Return a trash root within the /data mount point. Path dataTestPath = new Path("/data/dir/file"); - Path dataTrashRoot = new Path(targetTestRoot, - "data/" + TRASH_PREFIX + "/" + ugi.getShortUserName()); + Path dataTrashRoot = fsView2.makeQualified( + new Path("/data/" + TRASH_PREFIX + "/" + ugi.getShortUserName())); Assert.assertEquals(dataTrashRoot, fsView2.getTrashRoot(dataTestPath)); - // Case 2: turn off the CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH flag. - // Return a trash root in user home dir. + // Case 2: path p not found in mount table. + // Return a trash root in fallback FS. Path nonExistentPath = new Path("/nonExistentDir/nonExistentFile"); - Path userTrashRoot = new Path(fsTarget.getHomeDirectory(), TRASH_PREFIX); - conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, false); - fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); - Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(dataTestPath)); + Path expectedTrash = + fsView2.makeQualified(getTrashRootInFallBackFS()); + Assert.assertEquals(expectedTrash, fsView2.getTrashRoot(nonExistentPath)); - // Case 3: viewFS without fallback. Expect exception for a nonExistent path + // Case 3: turn off the CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT flag. + // Return a trash root in user home dir. + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, false); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + Path targetFSUserHomeTrashRoot = fsTarget.makeQualified( + new Path(fsTarget.getHomeDirectory(), TRASH_PREFIX)); + Assert.assertEquals(targetFSUserHomeTrashRoot, + fsView2.getTrashRoot(dataTestPath)); + + // Case 4: viewFS without fallback. Expect exception for a nonExistent path conf2 = new Configuration(conf); fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); try { fsView2.getTrashRoot(nonExistentPath); } catch (NotInMountpointException ignored) { } - - // Case 4: path p is in the same mount point as targetFS.getTrashRoot(). - // Return targetFS.getTrashRoot() - // Use a new Configuration object, so that we can start with an empty - // mount table. This would avoid a conflict between the /user link in - // setupMountPoints() and homeDir we will need to setup for this test. - // default homeDir for hdfs is /user/. - Configuration conf3 = ViewFileSystemTestSetup.createConfig(); - Path homeDir = fsTarget.getHomeDirectory(); - String homeParentDir = homeDir.getParent().toUri().getPath(); - conf3.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); - ConfigUtil.addLink(conf3, homeParentDir, - new Path(targetTestRoot, homeParentDir).toUri()); - Path homeTestPath = new Path(homeDir.toUri().getPath(), "testuser/file"); - FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3); - Assert.assertEquals(userTrashRoot, fsView3.getTrashRoot(homeTestPath)); } /** * A mocked FileSystem which returns a deep trash dir. */ - static class MockTrashRootFS extends MockFileSystem { + static class DeepTrashRootMockFS extends MockFileSystem { public static final Path TRASH = - new Path("/mnt/very/deep/deep/trash/dir/.Trash"); + new Path("/vol/very/deep/deep/trash/dir/.Trash"); @Override public Path getTrashRoot(Path path) { @@ -1086,28 +1086,31 @@ abstract public class ViewFileSystemBaseTest { } /** - * Test a trash root that is inside a mount point for getTrashRoot. + * Test getTrashRoot that is very deep inside a mount point. */ @Test public void testTrashRootDeepTrashDir() throws IOException { Configuration conf2 = ViewFileSystemTestSetup.createConfig(); - conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); - conf2.setClass("fs.mocktrashfs.impl", MockTrashRootFS.class, + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true); + conf2.setClass("fs.mocktrashfs.impl", DeepTrashRootMockFS.class, FileSystem.class); - ConfigUtil.addLink(conf2, "/mnt", URI.create("mocktrashfs://mnt/path")); - Path testPath = new Path(MockTrashRootFS.TRASH, "projs/proj"); + ConfigUtil.addLink(conf2, "/mnt/datavol1", + URI.create("mocktrashfs://localhost/vol")); + Path testPath = new Path("/mnt/datavol1/projs/proj"); FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); - Assert.assertEquals(MockTrashRootFS.TRASH, fsView2.getTrashRoot(testPath)); + Path expectedTrash = fsView2.makeQualified( + new Path("/mnt/datavol1/very/deep/deep/trash/dir/.Trash")); + Assert.assertEquals(expectedTrash, fsView2.getTrashRoot(testPath)); } /** - * Test localized trash roots in getTrashRoots() for all users. + * Test getTrashRoots() for all users. */ @Test public void testTrashRootsAllUsers() throws IOException { Configuration conf2 = new Configuration(conf); - conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true); FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); // Case 1: verify correct trash roots from fsView and fsView2 @@ -1143,17 +1146,27 @@ abstract public class ViewFileSystemBaseTest { FileSystem fsView4 = FileSystem.get(FsConstants.VIEWFS_URI, conf4); int trashRootsNum4 = fsView4.getTrashRoots(true).size(); Assert.assertEquals(afterTrashRootsNum2 + 2, trashRootsNum4); + + // Case 4: test trash roots in fallback FS + fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user10")); + fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user11")); + fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user12")); + Configuration conf5 = new Configuration(conf2); + ConfigUtil.addLinkFallback(conf5, targetTestRoot.toUri()); + FileSystem fsView5 = FileSystem.get(FsConstants.VIEWFS_URI, conf5); + int trashRootsNum5 = fsView5.getTrashRoots(true).size(); + Assert.assertEquals(afterTrashRootsNum2 + 3, trashRootsNum5); } /** - * Test localized trash roots in getTrashRoots() for current user. + * Test getTrashRoots() for current user. */ @Test public void testTrashRootsCurrentUser() throws IOException { String currentUser = UserGroupInformation.getCurrentUser().getShortUserName(); Configuration conf2 = new Configuration(conf); - conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true); FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); int beforeTrashRootNum = fsView.getTrashRoots(false).size(); @@ -1169,6 +1182,14 @@ abstract public class ViewFileSystemBaseTest { int afterTrashRootsNum2 = fsView2.getTrashRoots(false).size(); Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum); Assert.assertEquals(beforeTrashRootNum2 + 2, afterTrashRootsNum2); + + // Test trash roots in fallback FS + Configuration conf3 = new Configuration(conf2); + fsTarget.mkdirs(new Path(targetTestRoot, TRASH_PREFIX + "/" + currentUser)); + ConfigUtil.addLinkFallback(conf3, targetTestRoot.toUri()); + FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3); + int trashRootsNum3 = fsView3.getTrashRoots(false).size(); + Assert.assertEquals(afterTrashRootsNum2 + 1, trashRootsNum3); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 83c2a58e5a3..4cc90f8e26e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -44,6 +44,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import org.junit.After; import org.junit.AfterClass; @@ -156,6 +157,13 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { return 2; } + @Override + Path getTrashRootInFallBackFS() throws IOException { + return new Path( + "/" + TRASH_PREFIX + "/" + UserGroupInformation.getCurrentUser() + .getShortUserName()); + } + //Rename should fail on across different fileSystems @Test public void testRenameAccorssFilesystem() throws IOException {