From 9e4f52d32319828c153a3ea658520b946988ae31 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Fri, 19 Jan 2018 14:18:19 +0800 Subject: [PATCH] HDFS-12973. RBF: Document global quota supporting in federation. Contributed by Yiqun Lin. --- .../hdfs/tools/federation/RouterAdmin.java | 3 +- .../src/site/markdown/HDFSCommands.md | 4 +++ .../src/site/markdown/HDFSRouterFederation.md | 29 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java index d5a2d77a3e0..398288d34ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java @@ -84,7 +84,8 @@ public class RouterAdmin extends Configured implements Tool { + "[-readonly] -owner -group -mode ]\n" + "\t[-rm ]\n" + "\t[-ls ]\n" - + "\t[-setQuota -ns -ss ]\n" + + "\t[-setQuota -nsQuota -ssQuota " + + "]\n" + "\t[-clrQuota \n"; System.out.println(usage); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index 58d9547773f..71fc8346450 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -428,12 +428,16 @@ Usage: [-add [-readonly] -owner -group -mode ] [-rm ] [-ls ] + [-setQuota -nsQuota -ssQuota ] + [-clrQuota ] | COMMAND\_OPTION | Description | |:---- |:---- | | `-add` *source* *nameservice* *destination* | Add a mount table entry or update if it exists. | | `-rm` *source* | Remove mount point of specified path. | | `-ls` *path* | List mount points under specified path. | +| `-setQuota` *path* `-nsQuota` *nsQuota* `-ssQuota` *ssQuota* | Set quota for specified path. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html) for the quota detail. | +| `-clrQuota` *path* | Clear quota of given mount point. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html) for the quota detail. | The commands for managing Router-based federation. See [Mount table management](./HDFSRouterFederation.html#Mount_table_management) for more info. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md index cd3f437a10d..75798a19fed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md @@ -127,6 +127,11 @@ Examples users may encounter include the following. * Copy file/folder in two different nameservices. * Write into a file/folder being rebalanced. +### Quota management +Federation supports and controls global quota at mount table level. +For performance reasons, the Router caches the quota usage and updates it periodically. These quota usage values +will be used for quota-verification during each WRITE RPC call invoked in RouterRPCSever. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html) +for the quota detail. ### State Store The (logically centralized, but physically distributed) State Store maintains: @@ -199,6 +204,21 @@ Mount table permission can be set by following command: The option mode is UNIX-style permissions for the mount table. Permissions are specified in octal, e.g. 0755. By default, this is set to 0755. +Router-based federation supports global quota at mount table level. Mount table entries may spread multiple subclusters and the global quota will be +accounted across these subclusters. + +The federation admin tool supports setting quotas for specified mount table entries: + + [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -setQuota /path -nsQuota 100 -ssQuota 1024 + +The above command means that we allow the path to have a maximum of 100 file/directories and use at most 1024 bytes storage space. The parameter for *ssQuota* +supports multiple size-unit suffix (e.g. 1k is 1KB, 5m is 5MB). If no suffix is specified then bytes is assumed. + +Ls command will show below information for each mount table entry: + + Source Destinations Owner Group Mode Quota/Usage + /path ns0->/path root supergroup rwxr-xr-x [NsQuota: 50/0, SsQuota: 100 B/0 B] + Client configuration -------------------- @@ -322,6 +342,15 @@ Monitor the namenodes in the subclusters for forwarding the client requests. | dfs.federation.router.monitor.namenode | | The identifier of the namenodes to monitor and heartbeat. | | dfs.federation.router.monitor.localnamenode.enable | `true` | If `true`, the Router should monitor the namenode in the local machine. | +### Quota + +Global quota supported in federation. + +| Property | Default | Description| +|:---- |:---- |:---- | +| dfs.federation.router.quota.enable | `false` | If `true`, the quota system enabled in the Router. | +| dfs.federation.router.quota-cache.update.interval | 60s | How often the Router updates quota cache. This setting supports multiple time unit suffixes. If no suffix is specified then milliseconds is assumed. | + Metrics -------