From fa082e1a0a32012de1befd38e34c18c7af303006 Mon Sep 17 00:00:00 2001 From: cnauroth Date: Thu, 14 May 2015 14:55:58 -0700 Subject: [PATCH] HADOOP-11713. ViewFileSystem should support snapshot methods. Contributed by Rakesh R. (cherry picked from commit 09fe16f166392a99e1e54001a9112c6a4632dfc8) --- .../hadoop-common/CHANGES.txt | 3 ++ .../hadoop/fs/viewfs/ChRootedFileSystem.java | 17 ++++++ .../apache/hadoop/fs/viewfs/ChRootedFs.java | 19 ++++++- .../hadoop/fs/viewfs/ViewFileSystem.java | 48 ++++++++++++++++- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 51 ++++++++++++++++-- .../fs/viewfs/TestChRootedFileSystem.java | 52 +++++++++++++++++++ .../hadoop/fs/viewfs/TestChRootedFs.java | 41 +++++++++++++++ .../fs/viewfs/ViewFileSystemBaseTest.java | 20 +++++++ .../hadoop/fs/viewfs/ViewFsBaseTest.java | 21 ++++++++ 9 files changed, 267 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e43540bb67e..412719dbdaa 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -95,6 +95,9 @@ Release 2.8.0 - UNRELEASED HADOOP-9723. Improve error message when hadoop archive output path already exists. (Jean-Baptiste Onofré and Yongjun Zhang via aajisak) + HADOOP-11713. ViewFileSystem should support snapshot methods. + (Rakesh R via cnauroth) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index 18e2391ee2e..f7a93e78183 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -363,6 +363,23 @@ class ChRootedFileSystem extends FilterFileSystem { super.removeXAttr(fullPath(path), name); } + @Override + public Path createSnapshot(Path path, String name) throws IOException { + return super.createSnapshot(fullPath(path), name); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + super.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName); + } + + @Override + public void deleteSnapshot(Path snapshotDir, String snapshotName) + throws IOException { + super.deleteSnapshot(fullPath(snapshotDir), snapshotName); + } + @Override public Path resolvePath(final Path p) throws IOException { return super.resolvePath(fullPath(p)); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index 68e756a81b5..a05a7000f5b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -361,7 +361,24 @@ class ChRootedFs extends AbstractFileSystem { } @Override - public void setVerifyChecksum(final boolean verifyChecksum) + public Path createSnapshot(Path path, String name) throws IOException { + return myFs.createSnapshot(fullPath(path), name); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + myFs.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName); + } + + @Override + public void deleteSnapshot(Path snapshotDir, String snapshotName) + throws IOException { + myFs.deleteSnapshot(fullPath(snapshotDir), snapshotName); + } + + @Override + public void setVerifyChecksum(final boolean verifyChecksum) throws IOException, UnresolvedLinkException { myFs.setVerifyChecksum(verifyChecksum); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 43fe23fb82a..860506401bf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -726,7 +726,32 @@ public class ViewFileSystem extends FileSystem { } return result; } - + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), + true); + return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), + true); + res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName, + snapshotNewName); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), + true); + res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName); + } + /* * An instance of this class represents an internal dir of the viewFs * that is internal dir of the mount table. @@ -1020,5 +1045,26 @@ public class ViewFileSystem extends FileSystem { checkPathIsSlash(path); throw readOnlyMountTable("removeXAttr", path); } + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("createSnapshot", path); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("renameSnapshot", path); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("deleteSnapshot", path); + } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 975496aa42b..a23aa869ee3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -621,7 +621,8 @@ public class ViewFs extends AbstractFileSystem { @Override public boolean isValidName(String src) { - // Prefix validated at mount time and rest of path validated by mount target. + // Prefix validated at mount time and rest of path validated by mount + // target. return true; } @@ -714,8 +715,31 @@ public class ViewFs extends AbstractFileSystem { fsState.resolve(getUriPath(path), true); res.targetFileSystem.removeXAttr(res.remainingPath, name); } - - + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + InodeTree.ResolveResult res = fsState.resolve( + getUriPath(path), true); + return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + InodeTree.ResolveResult res = fsState.resolve( + getUriPath(path), true); + res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName, + snapshotNewName); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) throws IOException { + InodeTree.ResolveResult res = fsState.resolve( + getUriPath(path), true); + res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName); + } + /* * An instance of this class represents an internal dir of the viewFs * ie internal dir of the mount table. @@ -1025,5 +1049,26 @@ public class ViewFs extends AbstractFileSystem { checkPathIsSlash(path); throw readOnlyMountTable("removeXAttr", path); } + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("createSnapshot", path); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("renameSnapshot", path); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("deleteSnapshot", path); + } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java index a13ee8d070c..e4a1ac9729a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java @@ -416,4 +416,56 @@ public class TestChRootedFileSystem { @Override public void initialize(URI name, Configuration conf) throws IOException {} } + + @Test(timeout = 30000) + public void testCreateSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path("/a/b/snapPath"); + + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI chrootUri = URI.create("mockfs://foo/a/b"); + ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); + FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) + .getRawFileSystem(); + + chrootFs.createSnapshot(snapRootPath, "snap1"); + verify(mockFs).createSnapshot(chRootedSnapRootPath, "snap1"); + } + + @Test(timeout = 30000) + public void testDeleteSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path("/a/b/snapPath"); + + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI chrootUri = URI.create("mockfs://foo/a/b"); + ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); + FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) + .getRawFileSystem(); + + chrootFs.deleteSnapshot(snapRootPath, "snap1"); + verify(mockFs).deleteSnapshot(chRootedSnapRootPath, "snap1"); + } + + @Test(timeout = 30000) + public void testRenameSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path("/a/b/snapPath"); + + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI chrootUri = URI.create("mockfs://foo/a/b"); + ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); + FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) + .getRawFileSystem(); + + chrootFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName"); + verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName", + "snapNewName"); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java index e639291c65b..20825e312c9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java @@ -327,4 +327,45 @@ public class TestChRootedFs { Assert.assertFalse(chRootedFs.isValidName("/test")); Mockito.verify(baseFs).isValidName("/chroot/test"); } + + @Test(timeout = 30000) + public void testCreateSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path( + Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath"); + AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem()); + ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo); + Mockito.doReturn(snapRootPath).when(baseFs) + .createSnapshot(chRootedSnapRootPath, "snap1"); + Assert.assertEquals(snapRootPath, + chRootedFs.createSnapshot(snapRootPath, "snap1")); + Mockito.verify(baseFs).createSnapshot(chRootedSnapRootPath, "snap1"); + } + + @Test(timeout = 30000) + public void testDeleteSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path( + Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath"); + AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem()); + ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo); + Mockito.doNothing().when(baseFs) + .deleteSnapshot(chRootedSnapRootPath, "snap1"); + chRootedFs.deleteSnapshot(snapRootPath, "snap1"); + Mockito.verify(baseFs).deleteSnapshot(chRootedSnapRootPath, "snap1"); + } + + @Test(timeout = 30000) + public void testRenameSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path( + Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath"); + AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem()); + ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo); + Mockito.doNothing().when(baseFs) + .renameSnapshot(chRootedSnapRootPath, "snapOldName", "snapNewName"); + chRootedFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName"); + Mockito.verify(baseFs).renameSnapshot(chRootedSnapRootPath, "snapOldName", + "snapNewName"); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 18769c278f5..7fad990fb8c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -865,4 +865,24 @@ public class ViewFileSystemBaseTest { fsView.removeXAttr(new Path("/internalDir"), "xattrName"); } + @Test(expected = AccessControlException.class) + public void testInternalCreateSnapshot1() throws IOException { + fsView.createSnapshot(new Path("/internalDir")); + } + + @Test(expected = AccessControlException.class) + public void testInternalCreateSnapshot2() throws IOException { + fsView.createSnapshot(new Path("/internalDir"), "snap1"); + } + + @Test(expected = AccessControlException.class) + public void testInternalRenameSnapshot() throws IOException { + fsView.renameSnapshot(new Path("/internalDir"), "snapOldName", + "snapNewName"); + } + + @Test(expected = AccessControlException.class) + public void testInternalDeleteSnapshot() throws IOException { + fsView.deleteSnapshot(new Path("/internalDir"), "snap1"); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 035b280249d..d8ab53911c1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -777,4 +777,25 @@ public class ViewFsBaseTest { public void testInternalRemoveXAttr() throws IOException { fcView.removeXAttr(new Path("/internalDir"), "xattrName"); } + + @Test(expected = AccessControlException.class) + public void testInternalCreateSnapshot1() throws IOException { + fcView.createSnapshot(new Path("/internalDir")); + } + + @Test(expected = AccessControlException.class) + public void testInternalCreateSnapshot2() throws IOException { + fcView.createSnapshot(new Path("/internalDir"), "snap1"); + } + + @Test(expected = AccessControlException.class) + public void testInternalRenameSnapshot() throws IOException { + fcView.renameSnapshot(new Path("/internalDir"), "snapOldName", + "snapNewName"); + } + + @Test(expected = AccessControlException.class) + public void testInternalDeleteSnapshot() throws IOException { + fcView.deleteSnapshot(new Path("/internalDir"), "snap1"); + } }