From 523ff816246322e7376c52b3c8df92c0d6e4f6ef Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Thu, 6 Apr 2023 19:44:47 +0800 Subject: [PATCH] HDFS-16952. Support getLinkTarget API in WebHDFS (#5517) Co-authored-by: Zhtttylz Reviewed-by: Shilun Fan Signed-off-by: Shilun Fan --- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 13 +++++++++ .../hadoop/hdfs/web/resources/GetOpParam.java | 1 + .../router/RouterWebHdfsMethods.java | 1 + .../web/resources/NamenodeWebHdfsMethods.java | 5 ++++ .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 17 +++++++++++ .../apache/hadoop/hdfs/web/TestWebHDFS.java | 28 +++++++++++++++++++ 6 files changed, 65 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index f0774e98d1f..615cf3bd7c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -2147,6 +2147,19 @@ public class WebHdfsFileSystem extends FileSystem }.run(); } + @Override + public Path getLinkTarget(Path f) throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_LINK_TARGET); + final HttpOpParam.Op op = GetOpParam.Op.GETLINKTARGET; + return new FsPathResponseRunner(op, f) { + @Override + Path decodeResponse(Map json) { + return new Path((String) json.get(Path.class.getSimpleName())); + } + }.run(); + } + @VisibleForTesting InetSocketAddress[] getResolvedNNAddr() { return nnAddrs; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index 14938c3c45b..89979295c79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -64,6 +64,7 @@ public class GetOpParam extends HttpOpParam { GETSNAPSHOTDIFF(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTDIFFLISTING(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTTABLEDIRECTORYLIST(false, HttpURLConnection.HTTP_OK), + GETLINKTARGET(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTLIST(false, HttpURLConnection.HTTP_OK); final boolean redirect; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java index a66953b1bd7..477a59941fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java @@ -385,6 +385,7 @@ public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods { case GETXATTRS: case LISTXATTRS: case CHECKACCESS: + case GETLINKTARGET: { return super.get(ugi, delegation, username, doAsUser, fullpath, op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index a3250c213ca..4b3b53731ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -1383,6 +1383,11 @@ public class NamenodeWebHdfsMethods { final String js = JsonUtil.toJsonString(snapshotList); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } + case GETLINKTARGET: { + String target = cp.getLinkTarget(fullpath); + final String js = JsonUtil.toJsonString("Path", target); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } default: throw new UnsupportedOperationException(op + " is not supported"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md index 5e5924ad36e..f84018ae821 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md @@ -58,6 +58,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop * [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileBlockLocations) * [`GETECPOLICY`](#Get_EC_Policy) (see [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).getErasureCodingPolicy) * [`GETSERVERDEFAULTS`](#Get_Server_Defaults) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getServerDefaults) + * [`GETLINKTARGET`](#Get_Link_Target) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getLinkTarget) * HTTP PUT * [`CREATE`](#Create_and_Write_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).create) * [`MKDIRS`](#Make_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).mkdirs) @@ -1139,6 +1140,22 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).access See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getServerDefaults +### Get Link Target + +* Submit a HTTP GET request. + + curl -i "http://:/webhdfs/v1/?op=GETLINKTARGET" + + The client receives a response with a [`Path` JSON object](#Path_JSON_Schema): + + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + + {"Path": "/user/username/targetFile"} + +See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getLinkTarget + Storage Policy Operations ------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index c4f53b05615..8f4759d8e30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -2202,6 +2202,34 @@ public class TestWebHDFS { cluster.shutdown(); } } + + @Test + public void testLinkTarget() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + try { + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(3) + .build(); + cluster.waitActive(); + + final WebHdfsFileSystem webHdfs = + WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsConstants.WEBHDFS_SCHEME); + + // Symbolic link + Path root = new Path("/webHdfsTest/"); + Path targetFile = new Path(root, "debug.log"); + FileSystemTestHelper.createFile(webHdfs, targetFile); + + Path symLink = new Path(root, "debug.link"); + + webHdfs.createSymlink(targetFile, symLink, false); + assertEquals(webHdfs.getLinkTarget(symLink), targetFile); + } finally { + cluster.shutdown(); + } + } + /** * Get FileStatus JSONObject from ListStatus response. */