HDFS-12972. RBF: Display mount table quota info in Web UI and admin command. Contributed by Yiqun Lin.

(cherry picked from commit 9afb8025d6)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java

@@ -19,14 +19,20 @@ package org.apache.hadoop.hdfs.server.federation.router;
 
 import org.apache.hadoop.fs.QuotaUsage;
 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
 import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
 import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * The subclass of {@link QuotaUsage} used in Router-based federation.
  */
 public final class RouterQuotaUsage extends QuotaUsage {
+
+  /** Default quota usage count. */
+  public static final long QUOTA_USAGE_COUNT_DEFAULT = 0;
+
   private RouterQuotaUsage(Builder builder) {
     super(builder);
   }
@@ -85,4 +91,29 @@ public final class RouterQuotaUsage extends QuotaUsage {
       throw new DSQuotaExceededException(getSpaceQuota(), getSpaceConsumed());
     }
   }
+
+  @Override
+  public String toString() {
+    String nsQuota = String.valueOf(getQuota());
+    String nsCount = String.valueOf(getFileAndDirectoryCount());
+    if (getQuota() == HdfsConstants.QUOTA_DONT_SET) {
+      nsQuota = "-";
+      nsCount = "-";
+    }
+
+    String ssQuota = StringUtils.byteDesc(getSpaceQuota());
+    String ssCount = StringUtils.byteDesc(getSpaceConsumed());
+    if (getSpaceQuota() == HdfsConstants.QUOTA_DONT_SET) {
+      ssQuota = "-";
+      ssCount = "-";
+    }
+
+    StringBuilder str = new StringBuilder();
+    str.append("[NsQuota: ").append(nsQuota).append("/")
+        .append(nsCount);
+    str.append(", SsQuota: ").append(ssQuota)
+        .append("/").append(ssCount)
+        .append("]");
+    return str.toString();
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java

@@ -144,8 +144,10 @@ public abstract class MountTable extends BaseRecord {
 
     // Set quota for mount table
     RouterQuotaUsage quota = new RouterQuotaUsage.Builder()
-        .fileAndDirectoryCount(0).quota(HdfsConstants.QUOTA_DONT_SET)
+        .fileAndDirectoryCount(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
-        .spaceConsumed(0).spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
+        .quota(HdfsConstants.QUOTA_DONT_SET)
+        .spaceConsumed(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
+        .spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
     record.setQuota(quota);
 
     // Validate
@@ -325,6 +327,10 @@ public abstract class MountTable extends BaseRecord {
       sb.append("[mode:").append(this.getMode()).append("]");
     }
 
+    if (this.getQuota() != null) {
+      sb.append("[quota:").append(this.getQuota()).append("]");
+    }
+
     return sb.toString();
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java

@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProt
 import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.DestOrder;
 import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProtoOrBuilder;
 import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoteLocationProto;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto;
 import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
 import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
@@ -255,16 +256,22 @@ public class MountTablePBImpl extends MountTable implements PBRecord {
   @Override
   public RouterQuotaUsage getQuota() {
     MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
-    if (!proto.hasQuota()) {
+
-      return null;
+    long nsQuota = HdfsConstants.QUOTA_DONT_SET;
+    long nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
+    long ssQuota = HdfsConstants.QUOTA_DONT_SET;
+    long ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
+    if (proto.hasQuota()) {
+      QuotaUsageProto quotaProto = proto.getQuota();
+      nsQuota = quotaProto.getQuota();
+      nsCount = quotaProto.getFileAndDirectoryCount();
+      ssQuota = quotaProto.getSpaceQuota();
+      ssCount = quotaProto.getSpaceConsumed();
     }
 
-    QuotaUsageProto quotaProto = proto.getQuota();
     RouterQuotaUsage.Builder builder = new RouterQuotaUsage.Builder()
-        .fileAndDirectoryCount(quotaProto.getFileAndDirectoryCount())
+        .quota(nsQuota).fileAndDirectoryCount(nsCount).spaceQuota(ssQuota)
-        .quota(quotaProto.getQuota())
+        .spaceConsumed(ssCount);
-        .spaceConsumed(quotaProto.getSpaceConsumed())
-        .spaceQuota(quotaProto.getSpaceQuota());
     return builder.build();
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java

@@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.slf4j.Logger;
@@ -395,8 +396,8 @@ public class RouterAdmin extends Configured implements Tool {
   private static void printMounts(List<MountTable> entries) {
     System.out.println("Mount Table Entries:");
     System.out.println(String.format(
-        "%-25s %-25s %-25s %-25s %-25s",
+        "%-25s %-25s %-25s %-25s %-25s %-25s",
-        "Source", "Destinations", "Owner", "Group", "Mode"));
+        "Source", "Destinations", "Owner", "Group", "Mode", "Quota/Usage"));
     for (MountTable entry : entries) {
       StringBuilder destBuilder = new StringBuilder();
       for (RemoteLocation location : entry.getDestinations()) {
@@ -409,8 +410,10 @@ public class RouterAdmin extends Configured implements Tool {
       System.out.print(String.format("%-25s %-25s", entry.getSourcePath(),
           destBuilder.toString()));
 
-      System.out.println(String.format(" %-25s %-25s %-25s",
+      System.out.print(String.format(" %-25s %-25s %-25s",
           entry.getOwnerName(), entry.getGroupName(), entry.getMode()));
+
+      System.out.println(String.format(" %-25s", entry.getQuota()));
     }
   }
 
@@ -436,7 +439,8 @@ public class RouterAdmin extends Configured implements Tool {
       } else if (parameters[i].equals("-ssQuota")) {
         i++;
         try {
-          ssQuota = Long.parseLong(parameters[i]);
+          ssQuota = StringUtils.TraditionalBinaryPrefix
+              .string2long(parameters[i]);
         } catch (Exception e) {
           System.err.println("Cannot parse ssQuota: " + parameters[i]);
         }
@@ -499,11 +503,11 @@ public class RouterAdmin extends Configured implements Tool {
       // If nsQuota or ssQuota was unset, reset corresponding usage
       // value to zero.
       if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
-        nsCount = 0;
+        nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
       }
 
       if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
-        ssCount = 0;
+        ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
       }
 
       RouterQuotaUsage updatedQuota = new RouterQuotaUsage.Builder()

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html

@@ -338,6 +338,7 @@
       <th>Owner</th>
       <th>Group</th>
       <th>Permission</th>
+      <th>Quota/Usage</th>
       <th>Date Modified</th>
       <th>Date Created</th>
     </tr>
@@ -353,6 +354,7 @@
       <td>{ownerName}</td>
       <td>{groupName}</td>
       <td>{mode}</td>
+      <td>{quota}</td>
       <td>{dateModified}</td>
       <td>{dateCreated}</td>
     </tr>

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java

@@ -87,6 +87,7 @@ public class TestFederationMetrics extends TestMetricsBase {
           assertEquals(entry.getOwnerName(), json.getString("ownerName"));
           assertEquals(entry.getGroupName(), json.getString("groupName"));
           assertEquals(entry.getMode().toString(), json.getString("mode"));
+          assertEquals(entry.getQuota().toString(), json.getString("quota"));
           assertNotNullAndNotEmpty(json.getString("dateCreated"));
           assertNotNullAndNotEmpty(json.getString("dateModified"));
           match++;

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java

@@ -313,9 +313,11 @@ public class TestRouterAdminCLI {
     RouterQuotaUsage quotaUsage = mountTable.getQuota();
 
     // verify the default quota set
-    assertEquals(0, quotaUsage.getFileAndDirectoryCount());
+    assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
+        quotaUsage.getFileAndDirectoryCount());
     assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
-    assertEquals(0, quotaUsage.getSpaceConsumed());
+    assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
+        quotaUsage.getSpaceConsumed());
     assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
 
     long nsQuota = 50;
@@ -334,6 +336,19 @@ public class TestRouterAdminCLI {
     assertEquals(nsQuota, quotaUsage.getQuota());
     assertEquals(ssQuota, quotaUsage.getSpaceQuota());
 
+    // use quota string for setting ss quota
+    String newSsQuota = "2m";
+    argv = new String[] {"-setQuota", src, "-ssQuota", newSsQuota};
+    assertEquals(0, ToolRunner.run(admin, argv));
+
+    stateStore.loadCache(MountTableStoreImpl.class, true);
+    getResponse = client.getMountTableManager()
+        .getMountTableEntries(getRequest);
+    mountTable = getResponse.getEntries().get(0);
+    quotaUsage = mountTable.getQuota();
+    // verify if ss quota is correctly set
+    assertEquals(2 * 1024 * 1024, quotaUsage.getSpaceQuota());
+
     // test clrQuota command
     argv = new String[] {"-clrQuota", src};
     assertEquals(0, ToolRunner.run(admin, argv));