From 1e84e46f1621fe694f806bfc41d3b2a06c9500b6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Fri, 23 Feb 2018 16:56:29 -0800 Subject: [PATCH] HDFS-13052. WebHDFS: Add support for snasphot diff. Contributed by Lokesh Jain. --- .../hadoop/hdfs/DFSOpsCountStatistics.java | 1 + .../hdfs/protocol/SnapshotDiffReport.java | 4 ++ .../hadoop/hdfs/web/JsonUtilClient.java | 49 +++++++++++++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 15 ++++ .../hadoop/hdfs/web/resources/GetOpParam.java | 3 +- .../web/resources/NamenodeWebHdfsMethods.java | 33 +++++++-- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 33 +++++++++ .../apache/hadoop/hdfs/web/TestWebHDFS.java | 72 ++++++++++++++++++- 8 files changed, 201 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java index 4b2a7610a3e..bbd1bd71565 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java @@ -87,6 +87,7 @@ public class DFSOpsCountStatistics extends StorageStatistics { SET_STORAGE_POLICY("op_set_storagePolicy"), SET_TIMES(CommonStatisticNames.OP_SET_TIMES), SET_XATTR("op_set_xattr"), + GET_SNAPSHOT_DIFF("op_get_snapshot_diff"), TRUNCATE(CommonStatisticNames.OP_TRUNCATE), UNSET_STORAGE_POLICY("op_unset_storage_policy"); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index be3b94dfa62..8ee4ec77027 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -69,6 +69,10 @@ public class SnapshotDiffReport { } return null; } + + public static DiffType parseDiffType(String s){ + return DiffType.valueOf(s.toUpperCase()); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index c274c494e7f..2725e9cdf3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.StringUtils; @@ -693,4 +695,51 @@ class JsonUtilClient { encryptDataTransfer, trashInterval, type, keyProviderUri, storagepolicyId); } + + public static SnapshotDiffReport toSnapshotDiffReport(final Map json) { + if (json == null) { + return null; + } + Map m = + (Map) json.get(SnapshotDiffReport.class.getSimpleName()); + String snapshotRoot = (String) m.get("snapshotRoot"); + String fromSnapshot = (String) m.get("fromSnapshot"); + String toSnapshot = (String) m.get("toSnapshot"); + List diffList = + toDiffList(getList(m, "diffList")); + return new SnapshotDiffReport(snapshotRoot, fromSnapshot, toSnapshot, + diffList); + } + + private static List toDiffList( + List objs) { + if (objs == null) { + return null; + } + List diffList = + new ChunkedArrayList<>(); + for (int i = 0; i < objs.size(); i++) { + diffList.add(toDiffReportEntry((Map) objs.get(i))); + } + return diffList; + } + + private static SnapshotDiffReport.DiffReportEntry toDiffReportEntry( + Map json) { + if (json == null) { + return null; + } + SnapshotDiffReport.DiffType type = + SnapshotDiffReport.DiffType.parseDiffType((String) json.get("type")); + byte[] sourcePath = toByteArray((String) json.get("sourcePath")); + byte[] targetPath = toByteArray((String) json.get("targetPath")); + return new SnapshotDiffReport.DiffReportEntry(type, sourcePath, targetPath); + } + + private static byte[] toByteArray(String str) { + if (str == null) { + return null; + } + return DFSUtilClient.string2Bytes(str); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 0db5af74acc..45260f3b23e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -96,6 +96,7 @@ import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FileEncryptionInfoProto; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -1317,6 +1318,20 @@ public class WebHdfsFileSystem extends FileSystem new SnapshotNameParam(snapshotNewName)).run(); } + public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir, + final String fromSnapshot, final String toSnapshot) throws IOException { + storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF); + final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTDIFF; + return new FsPathResponseRunner(op, snapshotDir, + new OldSnapshotNameParam(fromSnapshot), + new SnapshotNameParam(toSnapshot)) { + @Override + SnapshotDiffReport decodeResponse(Map json) { + return JsonUtilClient.toSnapshotDiffReport(json); + } + }.run(); + } + @Override public boolean setReplication(final Path p, final short replication ) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index 4d4d9be70f4..7061d8265d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -47,7 +47,8 @@ public class GetOpParam extends HttpOpParam { CHECKACCESS(false, HttpURLConnection.HTTP_OK), LISTSTATUS_BATCH(false, HttpURLConnection.HTTP_OK), - GETSERVERDEFAULTS(false, HttpURLConnection.HTTP_OK); + GETSERVERDEFAULTS(false, HttpURLConnection.HTTP_OK), + GETSNAPSHOTDIFF(false, HttpURLConnection.HTTP_OK); final boolean redirect; final int expectedHttpResponseCode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index d1f16a32b6b..5cb958a3ec9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; @@ -922,6 +923,10 @@ public class NamenodeWebHdfsMethods { final ExcludeDatanodesParam excludeDatanodes, @QueryParam(FsActionParam.NAME) @DefaultValue(FsActionParam.DEFAULT) final FsActionParam fsAction, + @QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT) + final SnapshotNameParam snapshotName, + @QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT) + final OldSnapshotNameParam oldSnapshotName, @QueryParam(TokenKindParam.NAME) @DefaultValue(TokenKindParam.DEFAULT) final TokenKindParam tokenKind, @QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT) @@ -932,8 +937,9 @@ public class NamenodeWebHdfsMethods { final StartAfterParam startAfter ) throws IOException, InterruptedException { return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length, - renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction, - tokenKind, tokenService, noredirect, startAfter); + renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, + fsAction, snapshotName, oldSnapshotName, tokenKind, tokenService, + noredirect, startAfter); } /** Handle HTTP GET request. */ @@ -968,6 +974,10 @@ public class NamenodeWebHdfsMethods { final ExcludeDatanodesParam excludeDatanodes, @QueryParam(FsActionParam.NAME) @DefaultValue(FsActionParam.DEFAULT) final FsActionParam fsAction, + @QueryParam(SnapshotNameParam.NAME) @DefaultValue(SnapshotNameParam.DEFAULT) + final SnapshotNameParam snapshotName, + @QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT) + final OldSnapshotNameParam oldSnapshotName, @QueryParam(TokenKindParam.NAME) @DefaultValue(TokenKindParam.DEFAULT) final TokenKindParam tokenKind, @QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT) @@ -980,15 +990,15 @@ public class NamenodeWebHdfsMethods { init(ugi, delegation, username, doAsUser, path, op, offset, length, renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction, - tokenKind, tokenService, startAfter); + snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter); return doAs(ugi, new PrivilegedExceptionAction() { @Override public Response run() throws IOException, URISyntaxException { - return get(ugi, delegation, username, doAsUser, - path.getAbsolutePath(), op, offset, length, renewer, bufferSize, - xattrNames, xattrEncoding, excludeDatanodes, fsAction, tokenKind, - tokenService, noredirect, startAfter); + return get(ugi, delegation, username, doAsUser, path.getAbsolutePath(), + op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, + excludeDatanodes, fsAction, snapshotName, oldSnapshotName, + tokenKind, tokenService, noredirect, startAfter); } }); } @@ -1015,6 +1025,8 @@ public class NamenodeWebHdfsMethods { final XAttrEncodingParam xattrEncoding, final ExcludeDatanodesParam excludeDatanodes, final FsActionParam fsAction, + final SnapshotNameParam snapshotName, + final OldSnapshotNameParam oldSnapshotName, final TokenKindParam tokenKind, final TokenServiceParam tokenService, final NoRedirectParam noredirectParam, @@ -1185,6 +1197,13 @@ public class NamenodeWebHdfsMethods { return Response.ok(serverDefaultsResponse) .type(MediaType.APPLICATION_JSON).build(); } + case GETSNAPSHOTDIFF: { + SnapshotDiffReport diffReport = + cp.getSnapshotDiffReport(fullpath, oldSnapshotName.getValue(), + snapshotName.getValue()); + final String js = JsonUtil.toJsonString(diffReport); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } default: throw new UnsupportedOperationException(op + " is not supported"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index eae6adbbb69..095b9ac3426 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -494,4 +494,37 @@ public class JsonUtil { m.put("defaultStoragePolicyId", serverDefaults.getDefaultStoragePolicyId()); return m; } + + public static String toJsonString(SnapshotDiffReport diffReport) { + return toJsonString(SnapshotDiffReport.class.getSimpleName(), + toJsonMap(diffReport)); + } + + private static Object toJsonMap(SnapshotDiffReport diffReport) { + final Map m = new TreeMap(); + m.put("snapshotRoot", diffReport.getSnapshotRoot()); + m.put("fromSnapshot", diffReport.getFromSnapshot()); + m.put("toSnapshot", diffReport.getLaterSnapshotName()); + Object[] diffList = new Object[diffReport.getDiffList().size()]; + for (int i = 0; i < diffReport.getDiffList().size(); i++) { + diffList[i] = toJsonMap(diffReport.getDiffList().get(i)); + } + m.put("diffList", diffList); + return m; + } + + private static Object toJsonMap( + SnapshotDiffReport.DiffReportEntry diffReportEntry) { + final Map m = new TreeMap(); + m.put("type", diffReportEntry.getType()); + if (diffReportEntry.getSourcePath() != null) { + m.put("sourcePath", + DFSUtilClient.bytes2String(diffReportEntry.getSourcePath())); + } + if (diffReportEntry.getTargetPath() != null) { + m.put("targetPath", + DFSUtilClient.bytes2String(diffReportEntry.getTargetPath())); + } + return m; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 9a8c9fcf1ca..c94122eb45e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -73,6 +73,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -82,6 +83,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; +import static org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -579,7 +583,7 @@ public class TestWebHDFS { } /** - * Test snapshot creation through WebHdfs + * Test snapshot creation through WebHdfs. */ @Test public void testWebHdfsCreateSnapshot() throws Exception { @@ -665,6 +669,72 @@ public class TestWebHDFS { } } + /** + * Test snapshot diff through WebHdfs + */ + @Test + public void testWebHdfsSnapshotDiff() throws Exception { + MiniDFSCluster cluster = null; + final Configuration conf = WebHdfsTestUtil.createConf(); + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final WebHdfsFileSystem webHdfs = WebHdfsTestUtil + .getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME); + + final Path foo = new Path("/foo"); + dfs.mkdirs(foo); + Path file0 = new Path(foo, "file0"); + DFSTestUtil.createFile(dfs, file0, 100, (short) 1, 0); + Path file1 = new Path(foo, "file1"); + DFSTestUtil.createFile(dfs, file1, 100, (short) 1, 0); + Path file2 = new Path(foo, "file2"); + DFSTestUtil.createFile(dfs, file2, 100, (short) 1, 0); + + dfs.allowSnapshot(foo); + webHdfs.createSnapshot(foo, "s1"); + final Path s1path = SnapshotTestHelper.getSnapshotRoot(foo, "s1"); + Assert.assertTrue(webHdfs.exists(s1path)); + + Path file3 = new Path(foo, "file3"); + DFSTestUtil.createFile(dfs, file3, 100, (short) 1, 0); + DFSTestUtil.appendFile(dfs, file0, 100); + dfs.delete(file1, false); + Path file4 = new Path(foo, "file4"); + dfs.rename(file2, file4); + + webHdfs.createSnapshot(foo, "s2"); + SnapshotDiffReport diffReport = + webHdfs.getSnapshotDiffReport(foo, "s1", "s2"); + + Assert.assertEquals("/foo", diffReport.getSnapshotRoot()); + Assert.assertEquals("s1", diffReport.getFromSnapshot()); + Assert.assertEquals("s2", diffReport.getLaterSnapshotName()); + DiffReportEntry entry0 = + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")); + DiffReportEntry entry1 = + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file0")); + DiffReportEntry entry2 = + new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file1")); + DiffReportEntry entry3 = + new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("file2"), + DFSUtil.string2Bytes("file4")); + DiffReportEntry entry4 = + new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file3")); + Assert.assertTrue(diffReport.getDiffList().contains(entry0)); + Assert.assertTrue(diffReport.getDiffList().contains(entry1)); + Assert.assertTrue(diffReport.getDiffList().contains(entry2)); + Assert.assertTrue(diffReport.getDiffList().contains(entry3)); + Assert.assertTrue(diffReport.getDiffList().contains(entry4)); + Assert.assertEquals(diffReport.getDiffList().size(), 5); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + @Test public void testWebHdfsCreateNonRecursive() throws IOException, URISyntaxException { MiniDFSCluster cluster = null;