From 12617fad2eb32108412dac9ecee286de6641d060 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Mon, 18 Nov 2019 16:04:52 +0530 Subject: [PATCH] HDFS-14955. RBF: getQuotaUsage() on mount point should return global quota. Contributed by Jinglun. --- .../hdfs/server/federation/router/Quota.java | 17 ++++++-- .../router/RouterQuotaUpdateService.java | 2 +- .../federation/router/TestRouterQuota.java | 42 +++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java index ed69726e793..ee656083c7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java @@ -125,7 +125,7 @@ public class Quota { * @throws IOException If the quota system is disabled. */ public QuotaUsage getQuotaUsage(String path) throws IOException { - return aggregateQuota(getEachQuotaUsage(path)); + return aggregateQuota(path, getEachQuotaUsage(path)); } /** @@ -234,20 +234,26 @@ public class Quota { /** * Aggregate quota that queried from sub-clusters. + * @param path Federation path of the results. * @param results Quota query result. * @return Aggregated Quota. */ - QuotaUsage aggregateQuota(Map results) { + QuotaUsage aggregateQuota(String path, + Map results) throws IOException { long nsCount = 0; long ssCount = 0; long nsQuota = HdfsConstants.QUOTA_RESET; long ssQuota = HdfsConstants.QUOTA_RESET; boolean hasQuotaUnset = false; + boolean isMountEntry = isMountEntry(path); for (Map.Entry entry : results.entrySet()) { RemoteLocation loc = entry.getKey(); QuotaUsage usage = entry.getValue(); - if (usage != null) { + if (isMountEntry) { + nsCount += usage.getFileAndDirectoryCount(); + ssCount += usage.getSpaceConsumed(); + } else if (usage != null) { // If quota is not set in real FileSystem, the usage // value will return -1. if (usage.getQuota() == -1 && usage.getSpaceQuota() == -1) { @@ -266,6 +272,11 @@ public class Quota { } } + if (isMountEntry) { + QuotaUsage quota = getGlobalQuota(path); + nsQuota = quota.getQuota(); + ssQuota = quota.getSpaceQuota(); + } QuotaUsage.Builder builder = new QuotaUsage.Builder() .fileAndDirectoryCount(nsCount).spaceConsumed(ssCount); if (hasQuotaUnset) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java index 7982bc92f6e..e5d44727a9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java @@ -110,7 +110,7 @@ public class RouterQuotaUpdateService extends PeriodicService { Quota quotaModule = this.rpcServer.getQuotaModule(); Map usageMap = quotaModule.getEachQuotaUsage(src); - currentQuotaUsage = quotaModule.aggregateQuota(usageMap); + currentQuotaUsage = quotaModule.aggregateQuota(src, usageMap); remoteQuotaUsage.putAll(usageMap); } catch (IOException ioe) { LOG.error("Unable to get quota usage for " + src, ioe); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index 0b9185a936a..19f7ad67dc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -964,6 +964,48 @@ public class TestRouterQuota { assertEquals(-1, qu.getSpaceQuota()); } + @Test + public void testGetQuotaUsageOnMountPoint() throws Exception { + long nsQuota = 5; + long ssQuota = 3 * BLOCK_SIZE; + prepareGlobalQuotaTestMountTable(nsQuota, ssQuota); + + final FileSystem routerFs = routerContext.getFileSystem(); + // Verify getQuotaUsage() of the mount point /dir-1. + QuotaUsage quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1")); + assertEquals(nsQuota, quotaUsage.getQuota()); + assertEquals(ssQuota, quotaUsage.getSpaceQuota()); + // Verify getQuotaUsage() of the mount point /dir-1/dir-2. + quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-2")); + assertEquals(nsQuota, quotaUsage.getQuota()); + assertEquals(ssQuota * 2, quotaUsage.getSpaceQuota()); + // Verify getQuotaUsage() of the mount point /dir-1/dir-2/dir-3 + quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-2/dir-3")); + assertEquals(nsQuota, quotaUsage.getQuota()); + assertEquals(ssQuota * 2, quotaUsage.getSpaceQuota()); + // Verify getQuotaUsage() of the mount point /dir-4 + quotaUsage = routerFs.getQuotaUsage(new Path("/dir-4")); + assertEquals(-1, quotaUsage.getQuota()); + assertEquals(-1, quotaUsage.getSpaceQuota()); + + routerFs.mkdirs(new Path("/dir-1/dir-normal")); + try { + // Verify getQuotaUsage() of the normal path /dir-1/dir-normal. + quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-normal")); + assertEquals(-1, quotaUsage.getQuota()); + assertEquals(-1, quotaUsage.getSpaceQuota()); + + routerFs.setQuota(new Path("/dir-1/dir-normal"), 100, 200); + // Verify getQuotaUsage() of the normal path /dir-1/dir-normal. + quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-normal")); + assertEquals(100, quotaUsage.getQuota()); + assertEquals(200, quotaUsage.getSpaceQuota()); + } finally { + // clear normal path. + routerFs.delete(new Path("/dir-1/dir-normal"), true); + } + } + /** * Add three mount tables. * /dir-1 --> ns0---/dir-1 [nsQuota, ssQuota]