From a5690b29a7dd7af3a3b8f843a78f855982e87131 Mon Sep 17 00:00:00 2001 From: Xiao Chen Date: Fri, 8 Jun 2018 15:13:38 -0700 Subject: [PATCH] HDFS-13642. Creating a file with block size smaller than EC policy's cell size should fail. (cherry picked from commit cf4108313da83e28d07676078a33016ec8856ff6) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java --- .../server/namenode/FSDirErasureCodingOp.java | 23 ++++++++++++++---- .../server/namenode/FSDirWriteFileOp.java | 10 ++------ .../hdfs/server/namenode/FSNamesystem.java | 21 ++++++++++++---- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 5 ++-- .../hdfs/TestDFSStripedOutputStream.java | 16 ++++++++++++ .../hdfs/TestErasureCodingExerciseAPIs.java | 2 +- .../hdfs/TestErasureCodingPolicies.java | 2 +- .../src/test/resources/editsStored | Bin 7909 -> 7909 bytes .../src/test/resources/editsStored.xml | 2 +- 9 files changed, 58 insertions(+), 23 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index f6a40937272..920451db6bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.XAttr; @@ -354,16 +355,28 @@ final class FSDirErasureCodingOp { } /** - * Check if the file or directory has an erasure coding policy. + * Get the erasure coding policy information for specified path and policy + * name. If ec policy name is given, it will be parsed and the corresponding + * policy will be returned. Otherwise, get the policy from the parents of the + * iip. * * @param fsn namespace + * @param ecPolicyName the ec policy name * @param iip inodes in the path containing the file - * @return Whether the file or directory has an erasure coding policy. + * @return {@link ErasureCodingPolicy}, or null if no policy is found * @throws IOException */ - static boolean hasErasureCodingPolicy(final FSNamesystem fsn, - final INodesInPath iip) throws IOException { - return unprotectedGetErasureCodingPolicy(fsn, iip) != null; + static ErasureCodingPolicy getErasureCodingPolicy(FSNamesystem fsn, + String ecPolicyName, INodesInPath iip) throws IOException { + ErasureCodingPolicy ecPolicy; + if (!StringUtils.isEmpty(ecPolicyName)) { + ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicyByName( + fsn, ecPolicyName); + } else { + ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( + fsn, iip); + } + return ecPolicy; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 95b1dae39ae..2875708b72d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.base.Preconditions; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.AddBlockFlag; @@ -551,13 +550,8 @@ class FSDirWriteFileOp { boolean isStriped = false; ErasureCodingPolicy ecPolicy = null; if (!shouldReplicate) { - if (!StringUtils.isEmpty(ecPolicyName)) { - ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicyByName( - fsd.getFSNamesystem(), ecPolicyName); - } else { - ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( - fsd.getFSNamesystem(), existing); - } + ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy( + fsd.getFSNamesystem(), ecPolicyName, existing); if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) { isStriped = true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index e2542489a25..0c8a257ecc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2431,11 +2431,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, iip = FSDirWriteFileOp.resolvePathForStartFile( dir, pc, src, flag, createParent); - if (shouldReplicate || - (org.apache.commons.lang.StringUtils.isEmpty(ecPolicyName) && - !FSDirErasureCodingOp.hasErasureCodingPolicy(this, iip))) { - blockManager.verifyReplication(src, replication, clientMachine); - } if (blockSize < minBlockSize) { throw new IOException("Specified block size is less than configured" + @@ -2443,6 +2438,22 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + "): " + blockSize + " < " + minBlockSize); } + if (shouldReplicate) { + blockManager.verifyReplication(src, replication, clientMachine); + } else { + final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp + .getErasureCodingPolicy(this, ecPolicyName, iip); + if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) { + if (blockSize < ecPolicy.getCellSize()) { + throw new IOException("Specified block size (" + blockSize + + ") is less than the cell size (" + ecPolicy.getCellSize() + +") of the erasure coding policy (" + ecPolicy + ")."); + } + } else { + blockManager.verifyReplication(src, replication, clientMachine); + } + } + FileEncryptionInfo feInfo = null; if (!iip.isRaw() && provider != null) { EncryptionKeyInfo ezInfo = FSDirEncryptionZoneOp.getEncryptionKeyInfo( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index a0f346bcd96..37996ae18e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -1558,8 +1558,9 @@ public class DFSTestUtil { out.write("replicated".getBytes()); } - try (FSDataOutputStream out = filesystem.createFile( - new Path(ecDir, "RS-3-2")).ecPolicyName(ecPolicyRS32.getName()).build()) { + try (FSDataOutputStream out = filesystem + .createFile(new Path(ecDir, "RS-3-2")) + .ecPolicyName(ecPolicyRS32.getName()).blockSize(1024 * 1024).build()) { out.write("RS-3-2".getBytes()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 473557bf275..379b1c7f6e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -231,4 +232,19 @@ public class TestDFSStripedOutputStream { StripedFileTestUtil.checkData(fs, testPath, writeBytes, new ArrayList(), null, blockSize * dataBlocks); } + + @Test + public void testFileBlockSizeSmallerThanCellSize() throws Exception { + final Path path = new Path("testFileBlockSizeSmallerThanCellSize"); + final byte[] bytes = StripedFileTestUtil.generateBytes(cellSize * 2); + try { + DFSTestUtil.writeFile(fs, path, bytes, cellSize / 2); + fail("Creating a file with block size smaller than " + + "ec policy's cell size should fail"); + } catch (IOException expected) { + LOG.info("Caught expected exception", expected); + GenericTestUtils + .assertExceptionContains("less than the cell size", expected); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java index c63ba347f73..de59a1d71e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java @@ -71,7 +71,7 @@ public class TestErasureCodingExerciseAPIs { private DistributedFileSystem fs; private HdfsAdmin dfsAdmin; private FileSystemTestWrapper fsWrapper; - private static final int BLOCK_SIZE = 1 << 14; // 16k + private static final int BLOCK_SIZE = 1 << 20; // 1MB private ErasureCodingPolicy ecPolicy; private static ErasureCodingPolicy getEcPolicy() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 5c703c1723b..835d18f3a08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -64,7 +64,7 @@ public class TestErasureCodingPolicies { private Configuration conf; private MiniDFSCluster cluster; private DistributedFileSystem fs; - private static final int BLOCK_SIZE = 16 * 1024; + private static final int BLOCK_SIZE = 1024 * 1024; private ErasureCodingPolicy ecPolicy; private FSNamesystem namesystem; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index a0ae78eea82f1b65dec44c1d2fbc146e3397a06e..0382432dc86a725f50c1340c1692cea0ca1d6481 100644 GIT binary patch delta 28 kcmaEA`_y(rmMoJ1!{k6F-^n{=#aRxo6rQvBimV_b0Ggi)761SM delta 29 lcmaEA`_y(rmh9v_Ikw4OY@U;M%8IjWkl(Op^A%Y^MgX+H3qk+@ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index 7e1881c74a5..cc72e0dd84d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -1510,7 +1510,7 @@ 1 1512607204120 1512607204120 - 512 + 1048576 DFSClient_NONMAPREDUCE_-923924783_1 127.0.0.1 true