HDFS-11916. Extend TestErasureCodingPolicies/TestErasureCodingPolicyWithSnapshot with a random EC policy. Contributed by Takanobu Asanuma.

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java

@@ -61,15 +61,19 @@ public class TestErasureCodingPolicies {
   private MiniDFSCluster cluster;
   private DistributedFileSystem fs;
   private static final int BLOCK_SIZE = 1024;
-  private static final ErasureCodingPolicy EC_POLICY =
+  private ErasureCodingPolicy ecPolicy;
-      StripedFileTestUtil.getDefaultECPolicy();
   private FSNamesystem namesystem;
 
+  public ErasureCodingPolicy getEcPolicy() {
+    return StripedFileTestUtil.getDefaultECPolicy();
+  }
+
   @Rule
   public Timeout timeout = new Timeout(60 * 1000);
 
   @Before
   public void setupCluster() throws IOException {
+    ecPolicy = getEcPolicy();
     conf = new HdfsConfiguration();
     conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
     DFSTestUtil.enableAllECPolicies(conf);
@@ -100,8 +104,7 @@ public class TestErasureCodingPolicies {
     DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L);
 
     // set ec policy on dir
-    fs.setErasureCodingPolicy(dir,
+    fs.setErasureCodingPolicy(dir, ecPolicy.getName());
-        StripedFileTestUtil.getDefaultECPolicy().getName());
     // create a file which should be using ec
     final Path ecSubDir = new Path(dir, "ecSubDir");
     final Path ecFile = new Path(ecSubDir, "ecFile");
@@ -153,7 +156,7 @@ public class TestErasureCodingPolicies {
     fs.mkdir(testDir, FsPermission.getDirDefault());
 
     /* Normal creation of an erasure coding directory */
-    fs.setErasureCodingPolicy(testDir, EC_POLICY.getName());
+    fs.setErasureCodingPolicy(testDir, ecPolicy.getName());
 
     /* Verify files under the directory are striped */
     final Path ECFilePath = new Path(testDir, "foo");
@@ -169,7 +172,7 @@ public class TestErasureCodingPolicies {
     fs.mkdir(notEmpty, FsPermission.getDirDefault());
     final Path oldFile = new Path(notEmpty, "old");
     fs.create(oldFile);
-    fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName());
+    fs.setErasureCodingPolicy(notEmpty, ecPolicy.getName());
     final Path newFile = new Path(notEmpty, "new");
     fs.create(newFile);
     INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString());
@@ -181,10 +184,10 @@ public class TestErasureCodingPolicies {
     final Path dir1 = new Path("/dir1");
     final Path dir2 = new Path(dir1, "dir2");
     fs.mkdir(dir1, FsPermission.getDirDefault());
-    fs.setErasureCodingPolicy(dir1, EC_POLICY.getName());
+    fs.setErasureCodingPolicy(dir1, ecPolicy.getName());
     fs.mkdir(dir2, FsPermission.getDirDefault());
     try {
-      fs.setErasureCodingPolicy(dir2, EC_POLICY.getName());
+      fs.setErasureCodingPolicy(dir2, ecPolicy.getName());
     } catch (IOException e) {
       fail("Nested erasure coding policies are supported");
     }
@@ -193,7 +196,7 @@ public class TestErasureCodingPolicies {
     final Path fPath = new Path("/file");
     fs.create(fPath);
     try {
-      fs.setErasureCodingPolicy(fPath, EC_POLICY.getName());
+      fs.setErasureCodingPolicy(fPath, ecPolicy.getName());
       fail("Erasure coding policy on file");
     } catch (IOException e) {
       assertExceptionContains("erasure coding policy for a file", e);
@@ -213,11 +216,11 @@ public class TestErasureCodingPolicies {
     // Already set directory-level policies should still be in effect
     Path disabledPolicy = new Path(dir1, "afterDisabled");
     Assert.assertEquals("Dir does not have policy set",
-        EC_POLICY,
+        ecPolicy,
         fs.getErasureCodingPolicy(dir1));
     fs.create(disabledPolicy).close();
     Assert.assertEquals("File did not inherit dir's policy",
-        EC_POLICY,
+        ecPolicy,
         fs.getErasureCodingPolicy(disabledPolicy));
 
     // Also check loading disabled EC policies from fsimage
@@ -227,10 +230,10 @@ public class TestErasureCodingPolicies {
     cluster.restartNameNodes();
 
     Assert.assertEquals("Dir does not have policy set",
-        EC_POLICY,
+        ecPolicy,
         fs.getErasureCodingPolicy(dir1));
     Assert.assertEquals("File does not have policy set",
-        EC_POLICY,
+        ecPolicy,
         fs.getErasureCodingPolicy(disabledPolicy));
   }
 
@@ -240,8 +243,8 @@ public class TestErasureCodingPolicies {
     final Path dstECDir = new Path("/dstEC");
     fs.mkdir(srcECDir, FsPermission.getDirDefault());
     fs.mkdir(dstECDir, FsPermission.getDirDefault());
-    fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName());
+    fs.setErasureCodingPolicy(srcECDir, ecPolicy.getName());
-    fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName());
+    fs.setErasureCodingPolicy(dstECDir, ecPolicy.getName());
     final Path srcFile = new Path(srcECDir, "foo");
     fs.create(srcFile);
 
@@ -275,8 +278,7 @@ public class TestErasureCodingPolicies {
   public void testReplication() throws IOException {
     final Path testDir = new Path("/ec");
     fs.mkdir(testDir, FsPermission.getDirDefault());
-    fs.setErasureCodingPolicy(testDir,
+    fs.setErasureCodingPolicy(testDir, ecPolicy.getName());
-        StripedFileTestUtil.getDefaultECPolicy().getName());
     final Path fooFile = new Path(testDir, "foo");
     // create ec file with replication=0
     fs.create(fooFile, FsPermission.getFileDefault(), true,
@@ -330,10 +332,10 @@ public class TestErasureCodingPolicies {
   private void verifyErasureCodingInfo(
       String src, ErasureCodingPolicy usingECPolicy) throws IOException {
     HdfsFileStatus hdfsFileStatus = fs.getClient().getFileInfo(src);
-    ErasureCodingPolicy ecPolicy = hdfsFileStatus.getErasureCodingPolicy();
+    ErasureCodingPolicy actualPolicy = hdfsFileStatus.getErasureCodingPolicy();
-    assertNotNull(ecPolicy);
+    assertNotNull(actualPolicy);
     assertEquals("Actually used ecPolicy should be equal with target ecPolicy",
-        usingECPolicy, ecPolicy);
+        usingECPolicy, actualPolicy);
   }
 
   @Test
@@ -342,13 +344,13 @@ public class TestErasureCodingPolicies {
     ECSchema rsSchema = new ECSchema("rs", 4, 2);
     String policyName = "RS-4-2-128k";
     int cellSize = 128 * 1024;
-    ErasureCodingPolicy ecPolicy =
+    ErasureCodingPolicy invalidPolicy =
         new ErasureCodingPolicy(policyName, rsSchema, cellSize, (byte) -1);
     String src = "/ecDir4-2";
     final Path ecDir = new Path(src);
     try {
       fs.mkdir(ecDir, FsPermission.getDirDefault());
-      fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName());
+      fs.getClient().setErasureCodingPolicy(src, invalidPolicy.getName());
       fail("HadoopIllegalArgumentException should be thrown for"
           + "setting an invalid erasure coding policy");
     } catch (Exception e) {
@@ -429,8 +431,7 @@ public class TestErasureCodingPolicies {
     Path ecfile = new Path(ecdir, "ecfile");
     fs.setPermission(new Path("/"), new FsPermission((short)0777));
     userfs.mkdirs(ecdir);
-    final String ecPolicyName = StripedFileTestUtil.getDefaultECPolicy()
+    final String ecPolicyName = ecPolicy.getName();
-        .getName();
     useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
     assertEquals("Policy not present on dir",
         ecPolicyName,
@@ -537,12 +538,12 @@ public class TestErasureCodingPolicies {
     final Path filePath1 = new Path(dirPath, "file1");
 
     fs.mkdirs(dirPath);
-    fs.setErasureCodingPolicy(dirPath, EC_POLICY.getName());
+    fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
 
     // null EC policy name value means inheriting parent directory's policy
     fs.createFile(filePath0).build().close();
     ErasureCodingPolicy ecPolicyOnFile = fs.getErasureCodingPolicy(filePath0);
-    assertEquals(EC_POLICY, ecPolicyOnFile);
+    assertEquals(ecPolicy, ecPolicyOnFile);
 
     // Test illegal EC policy name
     final String illegalPolicyName = "RS-DEFAULT-1-2-64k";
@@ -560,7 +561,8 @@ public class TestErasureCodingPolicies {
     final ErasureCodingPolicy ecPolicyOnDir =
         SystemErasureCodingPolicies.getByID(
             SystemErasureCodingPolicies.RS_3_2_POLICY_ID);
-    ecPolicyOnFile = EC_POLICY;
+    ecPolicyOnFile = SystemErasureCodingPolicies.getByID(
+        SystemErasureCodingPolicies.RS_6_3_POLICY_ID);
     fs.setErasureCodingPolicy(dirPath, ecPolicyOnDir.getName());
     fs.createFile(filePath0).ecPolicyName(ecPolicyOnFile.getName())
         .build().close();
@@ -578,11 +580,11 @@ public class TestErasureCodingPolicies {
     final Path filePath = new Path(dirPath, "file");
 
     fs.mkdirs(dirPath);
-    fs.setErasureCodingPolicy(dirPath, EC_POLICY.getName());
+    fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
 
     final String ecPolicyName = "RS-10-4-64k";
     fs.createFile(filePath).build().close();
-    assertEquals(EC_POLICY, fs.getErasureCodingPolicy(filePath));
+    assertEquals(ecPolicy, fs.getErasureCodingPolicy(filePath));
     fs.delete(filePath, true);
 
     fs.createFile(filePath)

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPoliciesWithRandomECPolicy.java

@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This test extends TestErasureCodingPolicies to use a random (non-default) EC
+ * policy.
+ */
+public class TestErasureCodingPoliciesWithRandomECPolicy extends
+    TestErasureCodingPolicies {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      TestErasureCodingPoliciesWithRandomECPolicy.class);
+
+  private ErasureCodingPolicy ecPolicy;
+
+  public TestErasureCodingPoliciesWithRandomECPolicy() {
+    // If you want to debug this test with a specific ec policy, please use
+    // SystemErasureCodingPolicies class.
+    // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID);
+    ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
+    LOG.info("run {} with {}.", TestErasureCodingPoliciesWithRandomECPolicy
+        .class.getSuperclass().getSimpleName(), ecPolicy.getName());
+  }
+
+  @Override
+  public ErasureCodingPolicy getEcPolicy() {
+    return ecPolicy;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java

@@ -40,17 +40,21 @@ public class TestErasureCodingPolicyWithSnapshot {
   private Configuration conf;
 
   private final static int SUCCESS = 0;
-  private final ErasureCodingPolicy sysDefaultPolicy =
+  private ErasureCodingPolicy ecPolicy;
-      StripedFileTestUtil.getDefaultECPolicy();
+  private short groupSize;
-  private final short groupSize = (short) (
+
-      sysDefaultPolicy.getNumDataUnits() +
+  public ErasureCodingPolicy getEcPolicy() {
-          sysDefaultPolicy.getNumParityUnits());
+    return StripedFileTestUtil.getDefaultECPolicy();
+  }
 
   @Before
   public void setupCluster() throws IOException {
+    ecPolicy = getEcPolicy();
+    groupSize = (short) (ecPolicy.getNumDataUnits()
+        + ecPolicy.getNumParityUnits());
     conf = new HdfsConfiguration();
     conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
-        sysDefaultPolicy.getName());
+        ecPolicy.getName());
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
     cluster.waitActive();
     fs = cluster.getFileSystem();
@@ -77,12 +81,12 @@ public class TestErasureCodingPolicyWithSnapshot {
     fs.mkdirs(ecDir);
     fs.allowSnapshot(ecDirParent);
     // set erasure coding policy
-    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
+    fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
     DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
     String contents = DFSTestUtil.readFile(fs, ecFile);
     final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
     final Path snap1ECDir = new Path(snap1, ecDir.getName());
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         fs.getErasureCodingPolicy(snap1ECDir));
 
     // Now delete the dir which has erasure coding policy. Re-create the dir again, and
@@ -95,18 +99,18 @@ public class TestErasureCodingPolicyWithSnapshot {
         fs.getErasureCodingPolicy(snap2ECDir));
 
     // Make dir again with system default ec policy
-    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
+    fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
     final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
     final Path snap3ECDir = new Path(snap3, ecDir.getName());
     // Check that snap3's ECPolicy has the correct settings
     ErasureCodingPolicy ezSnap3 = fs.getErasureCodingPolicy(snap3ECDir);
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         ezSnap3);
 
     // Check that older snapshots still have the old ECPolicy settings
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         fs.getErasureCodingPolicy(snap1ECDir));
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         fs.getErasureCodingPolicy(snap2ECDir));
 
     // Verify contents of the snapshotted file
@@ -118,12 +122,12 @@ public class TestErasureCodingPolicyWithSnapshot {
     // Now delete the snapshots out of order and verify the EC policy
     // correctness
     fs.deleteSnapshot(ecDirParent, snap2.getName());
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         fs.getErasureCodingPolicy(snap1ECDir));
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         fs.getErasureCodingPolicy(snap3ECDir));
     fs.deleteSnapshot(ecDirParent, snap1.getName());
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         fs.getErasureCodingPolicy(snap3ECDir));
   }
 
@@ -136,9 +140,9 @@ public class TestErasureCodingPolicyWithSnapshot {
     fs.mkdirs(ecDir);
     fs.allowSnapshot(ecDir);
 
-    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
+    fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
     final Path snap1 = fs.createSnapshot(ecDir, "snap1");
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         fs.getErasureCodingPolicy(snap1));
   }
 
@@ -152,10 +156,10 @@ public class TestErasureCodingPolicyWithSnapshot {
     fs.allowSnapshot(ecDir);
 
     // set erasure coding policy
-    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
+    fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
     final Path snap1 = fs.createSnapshot(ecDir, "snap1");
     ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1);
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         ecSnap);
 
     // save namespace, restart namenode, and check ec policy correctness.
@@ -165,7 +169,7 @@ public class TestErasureCodingPolicyWithSnapshot {
     cluster.restartNameNode(true);
 
     ErasureCodingPolicy ecSnap1 = fs.getErasureCodingPolicy(snap1);
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         ecSnap1);
     assertEquals("Got unexpected ecSchema", ecSnap.getSchema(),
         ecSnap1.getSchema());
@@ -184,7 +188,7 @@ public class TestErasureCodingPolicyWithSnapshot {
     fs.allowSnapshot(ecDir);
 
     // set erasure coding policy
-    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
+    fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
     DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
     final Path snap1 = fs.createSnapshot(ecDir, "snap1");
 
@@ -197,7 +201,7 @@ public class TestErasureCodingPolicyWithSnapshot {
 
     assertNull("Got unexpected erasure coding policy",
         fs.getErasureCodingPolicy(snap1CopyECDir));
-    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+    assertEquals("Got unexpected erasure coding policy", ecPolicy,
         fs.getErasureCodingPolicy(snap1));
   }
 
@@ -212,7 +216,7 @@ public class TestErasureCodingPolicyWithSnapshot {
     fs.mkdirs(ecDir);
 
     // Set erasure coding policy
-    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
+    fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
     DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
 
     // Verify FileStatus for normal and EC files

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This test extends TestErasureCodingPolicyWithSnapshot to use a random
+ * (non-default) EC policy.
+ */
+public class TestErasureCodingPolicyWithSnapshotWithRandomECPolicy extends
+    TestErasureCodingPolicyWithSnapshot {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.class);
+
+  private ErasureCodingPolicy ecPolicy;
+
+  public TestErasureCodingPolicyWithSnapshotWithRandomECPolicy() {
+    // If you want to debug this test with a specific ec policy, please use
+    // SystemErasureCodingPolicies class.
+    // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID);
+    ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
+    LOG.info("run {} with {}.",
+        TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.class
+            .getSuperclass().getSimpleName(), ecPolicy.getName());
+  }
+
+  @Override
+  public ErasureCodingPolicy getEcPolicy() {
+    return ecPolicy;
+  }
+}