From 73fb75017e238e72c3162914f0db66e50139e199 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Mon, 19 Jun 2017 10:25:20 -0700 Subject: [PATCH] HDFS-11916. Extend TestErasureCodingPolicies/TestErasureCodingPolicyWithSnapshot with a random EC policy. Contributed by Takanobu Asanuma. --- .../hdfs/TestErasureCodingPolicies.java | 60 ++++++++++--------- ...asureCodingPoliciesWithRandomECPolicy.java | 48 +++++++++++++++ .../TestErasureCodingPolicyWithSnapshot.java | 50 +++++++++------- ...gPolicyWithSnapshotWithRandomECPolicy.java | 49 +++++++++++++++ 4 files changed, 155 insertions(+), 52 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPoliciesWithRandomECPolicy.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 4a4bed51c9f..f90a2f31289 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -61,15 +61,19 @@ public class TestErasureCodingPolicies { private MiniDFSCluster cluster; private DistributedFileSystem fs; private static final int BLOCK_SIZE = 1024; - private static final ErasureCodingPolicy EC_POLICY = - StripedFileTestUtil.getDefaultECPolicy(); + private ErasureCodingPolicy ecPolicy; private FSNamesystem namesystem; + public ErasureCodingPolicy getEcPolicy() { + return StripedFileTestUtil.getDefaultECPolicy(); + } + @Rule public Timeout timeout = new Timeout(60 * 1000); @Before public void setupCluster() throws IOException { + ecPolicy = getEcPolicy(); conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); DFSTestUtil.enableAllECPolicies(conf); @@ -100,8 +104,7 @@ public class TestErasureCodingPolicies { DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L); // set ec policy on dir - fs.setErasureCodingPolicy(dir, - StripedFileTestUtil.getDefaultECPolicy().getName()); + fs.setErasureCodingPolicy(dir, ecPolicy.getName()); // create a file which should be using ec final Path ecSubDir = new Path(dir, "ecSubDir"); final Path ecFile = new Path(ecSubDir, "ecFile"); @@ -153,7 +156,7 @@ public class TestErasureCodingPolicies { fs.mkdir(testDir, FsPermission.getDirDefault()); /* Normal creation of an erasure coding directory */ - fs.setErasureCodingPolicy(testDir, EC_POLICY.getName()); + fs.setErasureCodingPolicy(testDir, ecPolicy.getName()); /* Verify files under the directory are striped */ final Path ECFilePath = new Path(testDir, "foo"); @@ -169,7 +172,7 @@ public class TestErasureCodingPolicies { fs.mkdir(notEmpty, FsPermission.getDirDefault()); final Path oldFile = new Path(notEmpty, "old"); fs.create(oldFile); - fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName()); + fs.setErasureCodingPolicy(notEmpty, ecPolicy.getName()); final Path newFile = new Path(notEmpty, "new"); fs.create(newFile); INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString()); @@ -181,10 +184,10 @@ public class TestErasureCodingPolicies { final Path dir1 = new Path("/dir1"); final Path dir2 = new Path(dir1, "dir2"); fs.mkdir(dir1, FsPermission.getDirDefault()); - fs.setErasureCodingPolicy(dir1, EC_POLICY.getName()); + fs.setErasureCodingPolicy(dir1, ecPolicy.getName()); fs.mkdir(dir2, FsPermission.getDirDefault()); try { - fs.setErasureCodingPolicy(dir2, EC_POLICY.getName()); + fs.setErasureCodingPolicy(dir2, ecPolicy.getName()); } catch (IOException e) { fail("Nested erasure coding policies are supported"); } @@ -193,7 +196,7 @@ public class TestErasureCodingPolicies { final Path fPath = new Path("/file"); fs.create(fPath); try { - fs.setErasureCodingPolicy(fPath, EC_POLICY.getName()); + fs.setErasureCodingPolicy(fPath, ecPolicy.getName()); fail("Erasure coding policy on file"); } catch (IOException e) { assertExceptionContains("erasure coding policy for a file", e); @@ -213,11 +216,11 @@ public class TestErasureCodingPolicies { // Already set directory-level policies should still be in effect Path disabledPolicy = new Path(dir1, "afterDisabled"); Assert.assertEquals("Dir does not have policy set", - EC_POLICY, + ecPolicy, fs.getErasureCodingPolicy(dir1)); fs.create(disabledPolicy).close(); Assert.assertEquals("File did not inherit dir's policy", - EC_POLICY, + ecPolicy, fs.getErasureCodingPolicy(disabledPolicy)); // Also check loading disabled EC policies from fsimage @@ -227,10 +230,10 @@ public class TestErasureCodingPolicies { cluster.restartNameNodes(); Assert.assertEquals("Dir does not have policy set", - EC_POLICY, + ecPolicy, fs.getErasureCodingPolicy(dir1)); Assert.assertEquals("File does not have policy set", - EC_POLICY, + ecPolicy, fs.getErasureCodingPolicy(disabledPolicy)); } @@ -240,8 +243,8 @@ public class TestErasureCodingPolicies { final Path dstECDir = new Path("/dstEC"); fs.mkdir(srcECDir, FsPermission.getDirDefault()); fs.mkdir(dstECDir, FsPermission.getDirDefault()); - fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName()); - fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName()); + fs.setErasureCodingPolicy(srcECDir, ecPolicy.getName()); + fs.setErasureCodingPolicy(dstECDir, ecPolicy.getName()); final Path srcFile = new Path(srcECDir, "foo"); fs.create(srcFile); @@ -275,8 +278,7 @@ public class TestErasureCodingPolicies { public void testReplication() throws IOException { final Path testDir = new Path("/ec"); fs.mkdir(testDir, FsPermission.getDirDefault()); - fs.setErasureCodingPolicy(testDir, - StripedFileTestUtil.getDefaultECPolicy().getName()); + fs.setErasureCodingPolicy(testDir, ecPolicy.getName()); final Path fooFile = new Path(testDir, "foo"); // create ec file with replication=0 fs.create(fooFile, FsPermission.getFileDefault(), true, @@ -330,10 +332,10 @@ public class TestErasureCodingPolicies { private void verifyErasureCodingInfo( String src, ErasureCodingPolicy usingECPolicy) throws IOException { HdfsFileStatus hdfsFileStatus = fs.getClient().getFileInfo(src); - ErasureCodingPolicy ecPolicy = hdfsFileStatus.getErasureCodingPolicy(); - assertNotNull(ecPolicy); + ErasureCodingPolicy actualPolicy = hdfsFileStatus.getErasureCodingPolicy(); + assertNotNull(actualPolicy); assertEquals("Actually used ecPolicy should be equal with target ecPolicy", - usingECPolicy, ecPolicy); + usingECPolicy, actualPolicy); } @Test @@ -342,13 +344,13 @@ public class TestErasureCodingPolicies { ECSchema rsSchema = new ECSchema("rs", 4, 2); String policyName = "RS-4-2-128k"; int cellSize = 128 * 1024; - ErasureCodingPolicy ecPolicy = + ErasureCodingPolicy invalidPolicy = new ErasureCodingPolicy(policyName, rsSchema, cellSize, (byte) -1); String src = "/ecDir4-2"; final Path ecDir = new Path(src); try { fs.mkdir(ecDir, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName()); + fs.getClient().setErasureCodingPolicy(src, invalidPolicy.getName()); fail("HadoopIllegalArgumentException should be thrown for" + "setting an invalid erasure coding policy"); } catch (Exception e) { @@ -429,8 +431,7 @@ public class TestErasureCodingPolicies { Path ecfile = new Path(ecdir, "ecfile"); fs.setPermission(new Path("/"), new FsPermission((short)0777)); userfs.mkdirs(ecdir); - final String ecPolicyName = StripedFileTestUtil.getDefaultECPolicy() - .getName(); + final String ecPolicyName = ecPolicy.getName(); useradmin.setErasureCodingPolicy(ecdir, ecPolicyName); assertEquals("Policy not present on dir", ecPolicyName, @@ -537,12 +538,12 @@ public class TestErasureCodingPolicies { final Path filePath1 = new Path(dirPath, "file1"); fs.mkdirs(dirPath); - fs.setErasureCodingPolicy(dirPath, EC_POLICY.getName()); + fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); // null EC policy name value means inheriting parent directory's policy fs.createFile(filePath0).build().close(); ErasureCodingPolicy ecPolicyOnFile = fs.getErasureCodingPolicy(filePath0); - assertEquals(EC_POLICY, ecPolicyOnFile); + assertEquals(ecPolicy, ecPolicyOnFile); // Test illegal EC policy name final String illegalPolicyName = "RS-DEFAULT-1-2-64k"; @@ -560,7 +561,8 @@ public class TestErasureCodingPolicies { final ErasureCodingPolicy ecPolicyOnDir = SystemErasureCodingPolicies.getByID( SystemErasureCodingPolicies.RS_3_2_POLICY_ID); - ecPolicyOnFile = EC_POLICY; + ecPolicyOnFile = SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_6_3_POLICY_ID); fs.setErasureCodingPolicy(dirPath, ecPolicyOnDir.getName()); fs.createFile(filePath0).ecPolicyName(ecPolicyOnFile.getName()) .build().close(); @@ -578,11 +580,11 @@ public class TestErasureCodingPolicies { final Path filePath = new Path(dirPath, "file"); fs.mkdirs(dirPath); - fs.setErasureCodingPolicy(dirPath, EC_POLICY.getName()); + fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); final String ecPolicyName = "RS-10-4-64k"; fs.createFile(filePath).build().close(); - assertEquals(EC_POLICY, fs.getErasureCodingPolicy(filePath)); + assertEquals(ecPolicy, fs.getErasureCodingPolicy(filePath)); fs.delete(filePath, true); fs.createFile(filePath) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPoliciesWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPoliciesWithRandomECPolicy.java new file mode 100644 index 00000000000..9072b864dff --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPoliciesWithRandomECPolicy.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This test extends TestErasureCodingPolicies to use a random (non-default) EC + * policy. + */ +public class TestErasureCodingPoliciesWithRandomECPolicy extends + TestErasureCodingPolicies { + private static final Logger LOG = LoggerFactory.getLogger( + TestErasureCodingPoliciesWithRandomECPolicy.class); + + private ErasureCodingPolicy ecPolicy; + + public TestErasureCodingPoliciesWithRandomECPolicy() { + // If you want to debug this test with a specific ec policy, please use + // SystemErasureCodingPolicies class. + // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID); + ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy(); + LOG.info("run {} with {}.", TestErasureCodingPoliciesWithRandomECPolicy + .class.getSuperclass().getSimpleName(), ecPolicy.getName()); + } + + @Override + public ErasureCodingPolicy getEcPolicy() { + return ecPolicy; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java index 5549bf99400..fbeada67dc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java @@ -40,17 +40,21 @@ public class TestErasureCodingPolicyWithSnapshot { private Configuration conf; private final static int SUCCESS = 0; - private final ErasureCodingPolicy sysDefaultPolicy = - StripedFileTestUtil.getDefaultECPolicy(); - private final short groupSize = (short) ( - sysDefaultPolicy.getNumDataUnits() + - sysDefaultPolicy.getNumParityUnits()); + private ErasureCodingPolicy ecPolicy; + private short groupSize; + + public ErasureCodingPolicy getEcPolicy() { + return StripedFileTestUtil.getDefaultECPolicy(); + } @Before public void setupCluster() throws IOException { + ecPolicy = getEcPolicy(); + groupSize = (short) (ecPolicy.getNumDataUnits() + + ecPolicy.getNumParityUnits()); conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, - sysDefaultPolicy.getName()); + ecPolicy.getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build(); cluster.waitActive(); fs = cluster.getFileSystem(); @@ -77,12 +81,12 @@ public class TestErasureCodingPolicyWithSnapshot { fs.mkdirs(ecDir); fs.allowSnapshot(ecDirParent); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); + fs.setErasureCodingPolicy(ecDir, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); String contents = DFSTestUtil.readFile(fs, ecFile); final Path snap1 = fs.createSnapshot(ecDirParent, "snap1"); final Path snap1ECDir = new Path(snap1, ecDir.getName()); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, fs.getErasureCodingPolicy(snap1ECDir)); // Now delete the dir which has erasure coding policy. Re-create the dir again, and @@ -95,18 +99,18 @@ public class TestErasureCodingPolicyWithSnapshot { fs.getErasureCodingPolicy(snap2ECDir)); // Make dir again with system default ec policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); + fs.setErasureCodingPolicy(ecDir, ecPolicy.getName()); final Path snap3 = fs.createSnapshot(ecDirParent, "snap3"); final Path snap3ECDir = new Path(snap3, ecDir.getName()); // Check that snap3's ECPolicy has the correct settings ErasureCodingPolicy ezSnap3 = fs.getErasureCodingPolicy(snap3ECDir); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, ezSnap3); // Check that older snapshots still have the old ECPolicy settings - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, fs.getErasureCodingPolicy(snap1ECDir)); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, fs.getErasureCodingPolicy(snap2ECDir)); // Verify contents of the snapshotted file @@ -118,12 +122,12 @@ public class TestErasureCodingPolicyWithSnapshot { // Now delete the snapshots out of order and verify the EC policy // correctness fs.deleteSnapshot(ecDirParent, snap2.getName()); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, fs.getErasureCodingPolicy(snap1ECDir)); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, fs.getErasureCodingPolicy(snap3ECDir)); fs.deleteSnapshot(ecDirParent, snap1.getName()); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, fs.getErasureCodingPolicy(snap3ECDir)); } @@ -136,9 +140,9 @@ public class TestErasureCodingPolicyWithSnapshot { fs.mkdirs(ecDir); fs.allowSnapshot(ecDir); - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); + fs.setErasureCodingPolicy(ecDir, ecPolicy.getName()); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, fs.getErasureCodingPolicy(snap1)); } @@ -152,10 +156,10 @@ public class TestErasureCodingPolicyWithSnapshot { fs.allowSnapshot(ecDir); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); + fs.setErasureCodingPolicy(ecDir, ecPolicy.getName()); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, ecSnap); // save namespace, restart namenode, and check ec policy correctness. @@ -165,7 +169,7 @@ public class TestErasureCodingPolicyWithSnapshot { cluster.restartNameNode(true); ErasureCodingPolicy ecSnap1 = fs.getErasureCodingPolicy(snap1); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, ecSnap1); assertEquals("Got unexpected ecSchema", ecSnap.getSchema(), ecSnap1.getSchema()); @@ -184,7 +188,7 @@ public class TestErasureCodingPolicyWithSnapshot { fs.allowSnapshot(ecDir); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); + fs.setErasureCodingPolicy(ecDir, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); @@ -197,7 +201,7 @@ public class TestErasureCodingPolicyWithSnapshot { assertNull("Got unexpected erasure coding policy", fs.getErasureCodingPolicy(snap1CopyECDir)); - assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, + assertEquals("Got unexpected erasure coding policy", ecPolicy, fs.getErasureCodingPolicy(snap1)); } @@ -212,7 +216,7 @@ public class TestErasureCodingPolicyWithSnapshot { fs.mkdirs(ecDir); // Set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); + fs.setErasureCodingPolicy(ecDir, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); // Verify FileStatus for normal and EC files diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.java new file mode 100644 index 00000000000..8007ea3a3cd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This test extends TestErasureCodingPolicyWithSnapshot to use a random + * (non-default) EC policy. + */ +public class TestErasureCodingPolicyWithSnapshotWithRandomECPolicy extends + TestErasureCodingPolicyWithSnapshot { + private static final Logger LOG = LoggerFactory.getLogger( + TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.class); + + private ErasureCodingPolicy ecPolicy; + + public TestErasureCodingPolicyWithSnapshotWithRandomECPolicy() { + // If you want to debug this test with a specific ec policy, please use + // SystemErasureCodingPolicies class. + // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID); + ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy(); + LOG.info("run {} with {}.", + TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.class + .getSuperclass().getSimpleName(), ecPolicy.getName()); + } + + @Override + public ErasureCodingPolicy getEcPolicy() { + return ecPolicy; + } +}