From a8f1c7f542963f66849bcb2a06893c6a99cbe235 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 19 Jan 2017 00:24:11 -0800 Subject: [PATCH] HDFS-110268. Correctly reconstruct erasure coding file from FSImage. Contributed by SammiChen. --- .../server/namenode/FSImageFormatPBINode.java | 4 +- .../hdfs/server/namenode/TestFSImage.java | 80 ++++++++++++++++--- 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index f85978b68cc..fcc70122483 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -332,7 +332,9 @@ public final class FSImageFormatPBINode { short replication = (short) f.getReplication(); BlockType blockType = PBHelperClient.convert(f.getBlockType()); LoaderContext state = parent.getLoaderContext(); - ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy(); + ErasureCodingPolicy ecPolicy = (blockType == BlockType.STRIPED) ? + ErasureCodingPolicyManager.getPolicyByPolicyID((byte) replication) : + null; BlockInfo[] blocks = new BlockInfo[bp.size()]; for (int i = 0; i < bp.size(); ++i) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 4045320686f..339c5912906 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.File; @@ -31,6 +33,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.EnumSet; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; @@ -73,8 +76,9 @@ public class TestFSImage { private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ = "image-with-zero-block-size.tar.gz"; - private static final ErasureCodingPolicy testECPolicy - = ErasureCodingPolicyManager.getSystemDefaultPolicy(); + private static final ErasureCodingPolicy testECPolicy = + ErasureCodingPolicyManager.getPolicyByPolicyID( + HdfsConstants.RS_10_4_POLICY_ID); @Test public void testPersist() throws IOException { @@ -446,8 +450,8 @@ public class TestFSImage { /** * Ensure that FSImage supports BlockGroup. */ - @Test - public void testSupportBlockGroup() throws IOException { + @Test(timeout = 60000) + public void testSupportBlockGroup() throws Exception { final short GROUP_SIZE = (short) (testECPolicy.getNumDataUnits() + testECPolicy.getNumParityUnits()); final int BLOCK_SIZE = 8 * 1024 * 1024; @@ -459,32 +463,82 @@ public class TestFSImage { .build(); cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); - fs.getClient().getNamenode().setErasureCodingPolicy("/", testECPolicy); - Path file = new Path("/striped"); - FSDataOutputStream out = fs.create(file); - byte[] bytes = DFSTestUtil.generateSequentialBytes(0, BLOCK_SIZE); - out.write(bytes); - out.close(); + Path parentDir = new Path("/ec-10-4"); + Path childDir = new Path(parentDir, "ec-3-2"); + ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager + .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID); + // Create directories and files + fs.mkdirs(parentDir); + fs.mkdirs(childDir); + fs.setErasureCodingPolicy(parentDir, testECPolicy); + fs.setErasureCodingPolicy(childDir, ec32Policy); + Path file_10_4 = new Path(parentDir, "striped_file_10_4"); + Path file_3_2 = new Path(childDir, "striped_file_3_2"); + + // Write content to files + byte[] bytes = StripedFileTestUtil.generateBytes(BLOCK_SIZE); + DFSTestUtil.writeFile(fs, file_10_4, new String(bytes)); + DFSTestUtil.writeFile(fs, file_3_2, new String(bytes)); + + // Save namespace and restart NameNode fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fs.saveNamespace(); fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNodes(); fs = cluster.getFileSystem(); - assertTrue(fs.exists(file)); + assertTrue(fs.exists(file_10_4)); + assertTrue(fs.exists(file_3_2)); - // check the information of striped blocks + // check the information of file_10_4 FSNamesystem fsn = cluster.getNamesystem(); - INodeFile inode = fsn.dir.getINode(file.toString()).asFile(); + INodeFile inode = fsn.dir.getINode(file_10_4.toString()).asFile(); assertTrue(inode.isStriped()); BlockInfo[] blks = inode.getBlocks(); assertEquals(1, blks.length); assertTrue(blks[0].isStriped()); + assertEquals(testECPolicy.getId(), + fs.getErasureCodingPolicy(file_10_4).getId()); + assertEquals(testECPolicy.getId(), + ((BlockInfoStriped)blks[0]).getErasureCodingPolicy().getId()); assertEquals(testECPolicy.getNumDataUnits(), ((BlockInfoStriped) blks[0]).getDataBlockNum()); assertEquals(testECPolicy.getNumParityUnits(), ((BlockInfoStriped) blks[0]).getParityBlockNum()); + byte[] content = DFSTestUtil.readFileAsBytes(fs, file_10_4); + assertArrayEquals(bytes, content); + + + // check the information of file_3_2 + inode = fsn.dir.getINode(file_3_2.toString()).asFile(); + assertTrue(inode.isStriped()); + blks = inode.getBlocks(); + assertEquals(1, blks.length); + assertTrue(blks[0].isStriped()); + assertEquals(ec32Policy.getId(), + fs.getErasureCodingPolicy(file_3_2).getId()); + assertEquals(ec32Policy.getNumDataUnits(), + ((BlockInfoStriped) blks[0]).getDataBlockNum()); + assertEquals(ec32Policy.getNumParityUnits(), + ((BlockInfoStriped) blks[0]).getParityBlockNum()); + content = DFSTestUtil.readFileAsBytes(fs, file_3_2); + assertArrayEquals(bytes, content); + + // check the EC policy on parent Dir + ErasureCodingPolicy ecPolicy = + fsn.getErasureCodingPolicy(parentDir.toString()); + assertNotNull(ecPolicy); + assertEquals(testECPolicy.getId(), ecPolicy.getId()); + + // check the EC policy on child Dir + ecPolicy = fsn.getErasureCodingPolicy(childDir.toString()); + assertNotNull(ecPolicy); + assertEquals(ec32Policy.getId(), ecPolicy.getId()); + + // check the EC policy on root directory + ecPolicy = fsn.getErasureCodingPolicy("/"); + assertNull(ecPolicy); } finally { if (cluster != null) { cluster.shutdown();