HDFS-7776. Adding additional unit tests for Quota By Storage Type. (Contributed by Xiaoyu Yao)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -292,9 +292,6 @@ Trunk (Unreleased)
     HDFS-7721. The HDFS BlockScanner may run fast during the first hour
     (cmccabe)
 
-    HDFS-7751. Fix TestHDFSCLI for quota with storage type.  (Xiaoyu Yao
-    via szetszwo)
-
     HDFS-7670. HDFS Quota guide has typos, incomplete command lines
     (Brahma Reddy Battula via aw)
 
@@ -340,8 +337,8 @@ Release 2.7.0 - UNRELEASED
     HDFS-6133. Add a feature for replica pinning so that a pinned replica
     will not be moved by Balancer/Mover.  (zhaoyunjiong via szetszwo)
 
-    HDFS-7584. Enable Quota Support for Storage Types. (Xiaoyu Yao via
+    HDFS-7584. Enable Quota Support for Storage Types (See breakdown of
-    Arpit Agarwal)
+    tasks below)
 
   IMPROVEMENTS
 
@@ -948,6 +945,20 @@ Release 2.7.0 - UNRELEASED
     HDFS-7704. DN heartbeat to Active NN may be blocked and expire if
     connection to Standby NN continues to time out (Rushabh Shah via kihwal)
 
+    BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
+
+      HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
+      Protocol changes. (Xiaoyu Yao via Arpit Agarwal)
+
+      HDFS-7723. Quota By Storage Type namenode implemenation. (Xiaoyu Yao
+      via Arpit Agarwal)
+
+      HDFS-7751. Fix TestHDFSCLI for quota with storage type.  (Xiaoyu Yao
+      via szetszwo)
+
+      HDFS-7776. Adding additional unit tests for Quota By Storage Type.
+      (Xiaoyu Yao via Arpit Agarwal)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java

@@ -19,7 +19,6 @@
 
   import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
   import static org.junit.Assert.assertEquals;
-  import static org.junit.Assert.assertFalse;
   import static org.junit.Assert.assertTrue;
   import static org.junit.Assert.fail;
 
@@ -30,15 +29,18 @@
   import org.apache.hadoop.hdfs.*;
   import org.apache.hadoop.hdfs.protocol.HdfsConstants;
   import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
+  import org.apache.hadoop.test.GenericTestUtils;
   import org.junit.After;
   import org.junit.Before;
   import org.junit.Test;
 
+  import java.io.IOException;
+
 public class TestQuotaByStorageType {
 
   private static final int BLOCKSIZE = 1024;
   private static final short REPLICATION = 3;
-  static final long seed = 0L;
+  private static final long seed = 0L;
   private static final Path dir = new Path("/TestQuotaByStorageType");
 
   private Configuration conf;
@@ -219,7 +221,6 @@ public void testQuotaByStorageTypeWithFileCreateRename() throws Exception {
     // Verify space consumed and remaining quota
     long ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
         .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
-    ;
     assertEquals(file1Len, ssdConsumed);
 
     // move file from foo to bar
@@ -356,7 +357,6 @@ public void testQuotaByStorageTypeParentOnChildOff() throws Exception {
     assertTrue(fnode.isQuotaSet());
     long currentSSDConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
         .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
-    ;
     assertEquals(file1Len, currentSSDConsumed);
 
     // Create the 2nd file of size BLOCKSIZE under child directory and expect quota exceeded exception
@@ -399,11 +399,19 @@ public void testQuotaByStorageTypeParentOnChildOn() throws Exception {
     }
   }
 
+  /**
+   * Both traditional space quota and the storage type quota for SSD are set and
+   * not exceeded.
+   */
   @Test(timeout = 60000)
   public void testQuotaByStorageTypeWithTraditionalQuota() throws Exception {
     final Path foo = new Path(dir, "foo");
     dfs.mkdirs(foo);
+
+    dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+    dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 10);
     dfs.setQuota(foo, Long.MAX_VALUE - 1, REPLICATION * BLOCKSIZE * 10);
+
     INode fnode = fsdir.getINode4Write(foo.toString());
     assertTrue(fnode.isDirectory());
     assertTrue(fnode.isQuotaSet());
@@ -434,6 +442,76 @@ public void testQuotaByStorageTypeWithTraditionalQuota() throws Exception {
         counts.getDiskSpace());
   }
 
+  /**
+   * Both traditional space quota and the storage type quota for SSD are set and
+   * exceeded. expect DSQuotaExceededException is thrown as we check traditional
+   * space quota first and then storage type quota.
+   */
+  @Test(timeout = 60000)
+  public void testQuotaByStorageTypeAndTraditionalQuotaException1()
+      throws Exception {
+    testQuotaByStorageTypeOrTraditionalQuotaExceededCase(
+        4 * REPLICATION, 4, 5, REPLICATION);
+  }
+
+  /**
+   * Both traditional space quota and the storage type quota for SSD are set and
+   * SSD quota is exceeded but traditional space quota is not exceeded.
+   */
+  @Test(timeout = 60000)
+  public void testQuotaByStorageTypeAndTraditionalQuotaException2()
+      throws Exception {
+    testQuotaByStorageTypeOrTraditionalQuotaExceededCase(
+        5 * REPLICATION, 4, 5, REPLICATION);
+  }
+
+  /**
+   * Both traditional space quota and the storage type quota for SSD are set and
+   * traditional space quota is exceeded but SSD quota is not exceeded.
+   */
+  @Test(timeout = 60000)
+  public void testQuotaByStorageTypeAndTraditionalQuotaException3()
+      throws Exception {
+    testQuotaByStorageTypeOrTraditionalQuotaExceededCase(
+        4 * REPLICATION, 5, 5, REPLICATION);
+  }
+
+  private void testQuotaByStorageTypeOrTraditionalQuotaExceededCase(
+      long storageSpaceQuotaInBlocks, long ssdQuotaInBlocks,
+      long testFileLenInBlocks, short replication) throws Exception {
+    final String METHOD_NAME = GenericTestUtils.getMethodName();
+    final Path testDir = new Path(dir, METHOD_NAME);
+
+    dfs.mkdirs(testDir);
+    dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+    final long ssdQuota = BLOCKSIZE * ssdQuotaInBlocks;
+    final long storageSpaceQuota = BLOCKSIZE * storageSpaceQuotaInBlocks;
+
+    dfs.setQuota(testDir, Long.MAX_VALUE - 1, storageSpaceQuota);
+    dfs.setQuotaByStorageType(testDir, StorageType.SSD, ssdQuota);
+
+    INode testDirNode = fsdir.getINode4Write(testDir.toString());
+    assertTrue(testDirNode.isDirectory());
+    assertTrue(testDirNode.isQuotaSet());
+
+    Path createdFile = new Path(testDir, "created_file.data");
+    long fileLen = testFileLenInBlocks * BLOCKSIZE;
+
+    try {
+      DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16,
+          fileLen, BLOCKSIZE, replication, seed);
+      fail("Should have failed with DSQuotaExceededException or " +
+          "QuotaByStorageTypeExceededException ");
+    } catch (Throwable t) {
+      LOG.info("Got expected exception ", t);
+      long currentSSDConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
+          .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+      assertEquals(Math.min(ssdQuota, storageSpaceQuota/replication),
+          currentSSDConsumed);
+    }
+  }
+
   @Test(timeout = 60000)
   public void testQuotaByStorageTypeWithSnapshot() throws Exception {
     final Path sub1 = new Path(dir, "Sub1");
@@ -521,4 +599,105 @@ public void testQuotaByStorageTypeWithFileCreateTruncate() throws Exception {
         .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
     assertEquals(newFile1Len, ssdConsumed);
   }
+
+  @Test
+  public void testQuotaByStorageTypePersistenceInEditLog() throws IOException {
+    final String METHOD_NAME = GenericTestUtils.getMethodName();
+    final Path testDir = new Path(dir, METHOD_NAME);
+    Path createdFile1 = new Path(testDir, "created_file1.data");
+    dfs.mkdirs(testDir);
+
+    // set storage policy on testDir to ONESSD
+    dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+    // set quota by storage type on testDir
+    final long SSD_QUOTA = BLOCKSIZE * 4;
+    dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
+    INode testDirNode = fsdir.getINode4Write(testDir.toString());
+    assertTrue(testDirNode.isDirectory());
+    assertTrue(testDirNode.isQuotaSet());
+
+    // Create file of size 2 * BLOCKSIZE under testDir
+    long file1Len = BLOCKSIZE * 2;
+    int bufLen = BLOCKSIZE / 16;
+    DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
+
+    // Verify SSD consumed before namenode restart
+    long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
+        .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+    assertEquals(file1Len, ssdConsumed);
+
+    // Restart namenode to make sure the editlog is correct
+    cluster.restartNameNode(true);
+
+    INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
+    // Verify quota is still set
+    assertTrue(testDirNode.isDirectory());
+    assertTrue(testDirNode.isQuotaSet());
+
+    QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
+    assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
+    for (StorageType t: StorageType.getTypesSupportingQuota()) {
+      if (t != StorageType.SSD) {
+        assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
+      }
+    }
+
+    long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
+        .getDirectoryWithQuotaFeature()
+        .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+    assertEquals(file1Len, ssdConsumedAfterNNRestart);
+  }
+
+  @Test
+  public void testQuotaByStorageTypePersistenceInFsImage() throws IOException {
+    final String METHOD_NAME = GenericTestUtils.getMethodName();
+    final Path testDir = new Path(dir, METHOD_NAME);
+    Path createdFile1 = new Path(testDir, "created_file1.data");
+    dfs.mkdirs(testDir);
+
+    // set storage policy on testDir to ONESSD
+    dfs.setStoragePolicy(testDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+    // set quota by storage type on testDir
+    final long SSD_QUOTA = BLOCKSIZE * 4;
+    dfs.setQuotaByStorageType(testDir, StorageType.SSD, SSD_QUOTA);
+    INode testDirNode = fsdir.getINode4Write(testDir.toString());
+    assertTrue(testDirNode.isDirectory());
+    assertTrue(testDirNode.isQuotaSet());
+
+    // Create file of size 2 * BLOCKSIZE under testDir
+    long file1Len = BLOCKSIZE * 2;
+    int bufLen = BLOCKSIZE / 16;
+    DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
+
+    // Verify SSD consumed before namenode restart
+    long ssdConsumed = testDirNode.asDirectory().getDirectoryWithQuotaFeature()
+        .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+    assertEquals(file1Len, ssdConsumed);
+
+    // Restart the namenode with checkpoint to make sure fsImage is correct
+    dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+    dfs.saveNamespace();
+    dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+    cluster.restartNameNode(true);
+
+    INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
+    assertTrue(testDirNode.isDirectory());
+    assertTrue(testDirNode.isQuotaSet());
+
+    QuotaCounts qc = testDirNodeAfterNNRestart.getQuotaCounts();
+    assertEquals(SSD_QUOTA, qc.getTypeSpace(StorageType.SSD));
+    for (StorageType t: StorageType.getTypesSupportingQuota()) {
+      if (t != StorageType.SSD) {
+        assertEquals(HdfsConstants.QUOTA_RESET, qc.getTypeSpace(t));
+      }
+    }
+
+    long ssdConsumedAfterNNRestart = testDirNodeAfterNNRestart.asDirectory()
+        .getDirectoryWithQuotaFeature()
+        .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+    assertEquals(file1Len, ssdConsumedAfterNNRestart);
+
+  }
 }