From da16c9b3b40f9cbef0ea7d8cffc4c2c77fd1c447 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Tue, 13 Oct 2015 11:03:37 -0700 Subject: [PATCH] HDFS-8438. Erasure Coding: Allow concat striped files if they have the same ErasureCodingPolicy. Contributed by Walter Su. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSDirConcatOp.java | 9 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 117 +++++++++++++----- 3 files changed, 95 insertions(+), 34 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9751a3a0eb2..617c345196a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -832,6 +832,9 @@ Trunk (Unreleased) HDFS-9209. Erasure coding: Add apache license header in TestFileStatusWithECPolicy.java. (Surendra Singh Lilhore via jing9) + HDFS-8438. Erasure Coding: Allow concat striped files if they have the same + ErasureCodingPolicy. (Walter Su via jing9) + Release 2.8.0 - UNRELEASED NEW FEATURES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java index e8b2c544fbd..83782d5a9ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java @@ -154,10 +154,11 @@ class FSDirConcatOp { + " which is greater than the target file's preferred block size " + targetINode.getPreferredBlockSize()); } - // TODO currently we do not support concatenating EC files - if (srcINodeFile.isStriped()) { - throw new HadoopIllegalArgumentException("concat: the src file " + src - + " is with striped blocks"); + if(srcINodeFile.getErasureCodingPolicyID() != + targetINode.getErasureCodingPolicyID()) { + throw new HadoopIllegalArgumentException("Source file " + src + + " and target file " + targetIIP.getPath() + + " have different erasure coding policy"); } si.add(srcINodeFile); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index c1ae2c1bca1..4beb01ff8fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -28,14 +28,18 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.web.WebHdfsConstants; import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Random; import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize; import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs; @@ -61,8 +65,9 @@ public class TestWriteReadStripedFile { public void setup() throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); fs = cluster.getFileSystem(); + fs.mkdirs(new Path("/ec")); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", null); } @After @@ -74,108 +79,110 @@ public class TestWriteReadStripedFile { @Test public void testFileEmpty() throws Exception { - testOneFileUsingDFSStripedInputStream("/EmptyFile", 0); - testOneFileUsingDFSStripedInputStream("/EmptyFile2", 0, true); + testOneFileUsingDFSStripedInputStream("/ec/EmptyFile", 0); + testOneFileUsingDFSStripedInputStream("/ec/EmptyFile2", 0, true); } @Test public void testFileSmallerThanOneCell1() throws Exception { - testOneFileUsingDFSStripedInputStream("/SmallerThanOneCell", 1); - testOneFileUsingDFSStripedInputStream("/SmallerThanOneCell2", 1, true); + testOneFileUsingDFSStripedInputStream("/ec/SmallerThanOneCell", 1); + testOneFileUsingDFSStripedInputStream("/ec/SmallerThanOneCell2", 1, true); } @Test public void testFileSmallerThanOneCell2() throws Exception { - testOneFileUsingDFSStripedInputStream("/SmallerThanOneCell", cellSize - 1); - testOneFileUsingDFSStripedInputStream("/SmallerThanOneCell2", cellSize - 1, - true); + testOneFileUsingDFSStripedInputStream("/ec/SmallerThanOneCell", + cellSize - 1); + testOneFileUsingDFSStripedInputStream("/ec/SmallerThanOneCell2", + cellSize - 1, true); } @Test public void testFileEqualsWithOneCell() throws Exception { - testOneFileUsingDFSStripedInputStream("/EqualsWithOneCell", cellSize); - testOneFileUsingDFSStripedInputStream("/EqualsWithOneCell2", cellSize, true); + testOneFileUsingDFSStripedInputStream("/ec/EqualsWithOneCell", cellSize); + testOneFileUsingDFSStripedInputStream("/ec/EqualsWithOneCell2", + cellSize, true); } @Test public void testFileSmallerThanOneStripe1() throws Exception { - testOneFileUsingDFSStripedInputStream("/SmallerThanOneStripe", + testOneFileUsingDFSStripedInputStream("/ec/SmallerThanOneStripe", cellSize * dataBlocks - 1); - testOneFileUsingDFSStripedInputStream("/SmallerThanOneStripe2", + testOneFileUsingDFSStripedInputStream("/ec/SmallerThanOneStripe2", cellSize * dataBlocks - 1, true); } @Test public void testFileSmallerThanOneStripe2() throws Exception { - testOneFileUsingDFSStripedInputStream("/SmallerThanOneStripe", + testOneFileUsingDFSStripedInputStream("/ec/SmallerThanOneStripe", cellSize + 123); - testOneFileUsingDFSStripedInputStream("/SmallerThanOneStripe2", + testOneFileUsingDFSStripedInputStream("/ec/SmallerThanOneStripe2", cellSize + 123, true); } @Test public void testFileEqualsWithOneStripe() throws Exception { - testOneFileUsingDFSStripedInputStream("/EqualsWithOneStripe", + testOneFileUsingDFSStripedInputStream("/ec/EqualsWithOneStripe", cellSize * dataBlocks); - testOneFileUsingDFSStripedInputStream("/EqualsWithOneStripe2", + testOneFileUsingDFSStripedInputStream("/ec/EqualsWithOneStripe2", cellSize * dataBlocks, true); } @Test public void testFileMoreThanOneStripe1() throws Exception { - testOneFileUsingDFSStripedInputStream("/MoreThanOneStripe1", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanOneStripe1", cellSize * dataBlocks + 123); - testOneFileUsingDFSStripedInputStream("/MoreThanOneStripe12", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanOneStripe12", cellSize * dataBlocks + 123, true); } @Test public void testFileMoreThanOneStripe2() throws Exception { - testOneFileUsingDFSStripedInputStream("/MoreThanOneStripe2", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanOneStripe2", cellSize * dataBlocks + cellSize * dataBlocks + 123); - testOneFileUsingDFSStripedInputStream("/MoreThanOneStripe22", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanOneStripe22", cellSize * dataBlocks + cellSize * dataBlocks + 123, true); } @Test public void testLessThanFullBlockGroup() throws Exception { - testOneFileUsingDFSStripedInputStream("/LessThanFullBlockGroup", + testOneFileUsingDFSStripedInputStream("/ec/LessThanFullBlockGroup", cellSize * dataBlocks * (stripesPerBlock - 1) + cellSize); - testOneFileUsingDFSStripedInputStream("/LessThanFullBlockGroup2", + testOneFileUsingDFSStripedInputStream("/ec/LessThanFullBlockGroup2", cellSize * dataBlocks * (stripesPerBlock - 1) + cellSize, true); } @Test public void testFileFullBlockGroup() throws Exception { - testOneFileUsingDFSStripedInputStream("/FullBlockGroup", + testOneFileUsingDFSStripedInputStream("/ec/FullBlockGroup", blockSize * dataBlocks); - testOneFileUsingDFSStripedInputStream("/FullBlockGroup2", + testOneFileUsingDFSStripedInputStream("/ec/FullBlockGroup2", blockSize * dataBlocks, true); } @Test public void testFileMoreThanABlockGroup1() throws Exception { - testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup1", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanABlockGroup1", blockSize * dataBlocks + 123); - testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup12", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanABlockGroup12", blockSize * dataBlocks + 123, true); } @Test public void testFileMoreThanABlockGroup2() throws Exception { - testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup2", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanABlockGroup2", blockSize * dataBlocks + cellSize + 123); - testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup22", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanABlockGroup22", blockSize * dataBlocks + cellSize + 123, true); } @Test public void testFileMoreThanABlockGroup3() throws Exception { - testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup3", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanABlockGroup3", blockSize * dataBlocks * 3 + cellSize * dataBlocks + cellSize + 123); - testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup32", + testOneFileUsingDFSStripedInputStream("/ec/MoreThanABlockGroup32", blockSize * dataBlocks * 3 + cellSize * dataBlocks + cellSize + 123, true); } @@ -252,4 +259,54 @@ public class TestWriteReadStripedFile { StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, smallBuf); // webhdfs doesn't support bytebuffer read } + + @Test + public void testConcat() throws Exception { + final byte[] data = + StripedFileTestUtil.generateBytes(blockSize * dataBlocks * 10 + 234); + int totalLength = 0; + + Random r = new Random(); + Path target = new Path("/ec/testConcat_target"); + DFSTestUtil.writeFile(fs, target, Arrays.copyOfRange(data, 0, 123)); + totalLength += 123; + + int numFiles = 5; + Path[] srcs = new Path[numFiles]; + for (int i = 0; i < numFiles; i++) { + srcs[i] = new Path("/ec/testConcat_src_file_" + i); + int srcLength = r.nextInt(blockSize * dataBlocks * 2) + 1; + DFSTestUtil.writeFile(fs, srcs[i], + Arrays.copyOfRange(data, totalLength, totalLength + srcLength)); + totalLength += srcLength; + } + + fs.concat(target, srcs); + StripedFileTestUtil.verifyStatefulRead(fs, target, totalLength, + Arrays.copyOfRange(data, 0, totalLength), new byte[1024]); + } + + @Test + public void testConcatWithDifferentECPolicy() throws Exception { + final byte[] data = + StripedFileTestUtil.generateBytes(blockSize * dataBlocks); + Path nonECFile = new Path("/non_ec_file"); + DFSTestUtil.writeFile(fs, nonECFile, data); + Path target = new Path("/ec/non_ec_file"); + fs.rename(nonECFile, target); + + int numFiles = 2; + Path[] srcs = new Path[numFiles]; + for (int i = 0; i < numFiles; i++) { + srcs[i] = new Path("/ec/testConcat_src_file_"+i); + DFSTestUtil.writeFile(fs, srcs[i], data); + } + try { + fs.concat(target, srcs); + Assert.fail("non-ec file shouldn't concat with ec file"); + } catch (RemoteException e){ + Assert.assertTrue(e.getMessage() + .contains("have different erasure coding policy")); + } + } }