HDFS-4305. Add a configurable limit on number of blocks per file, and min block size. Contributed by Andrew Wang.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1477354 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers
parent
ce73b3b01f
commit
ce7e5565f4
|
|
@ -611,6 +611,9 @@ Release 2.0.5-beta - UNRELEASED
|
|||
|
||||
HDFS-4733. Make HttpFS username pattern configurable. (tucu via atm)
|
||||
|
||||
HDFS-4305. Add a configurable limit on number of blocks per file, and min
|
||||
block size. (Andrew Wang via atm)
|
||||
|
||||
Release 2.0.4-alpha - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
|||
|
|
@ -227,6 +227,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
|||
public static final int DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT = 0; // no limit
|
||||
public static final String DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY = "dfs.namenode.fs-limits.max-directory-items";
|
||||
public static final int DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT = 0; // no limit
|
||||
public static final String DFS_NAMENODE_MIN_BLOCK_SIZE_KEY = "dfs.namenode.fs-limits.min-block-size";
|
||||
public static final long DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT = 1024*1024;
|
||||
public static final String DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY = "dfs.namenode.fs-limits.max-blocks-per-file";
|
||||
public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 1024*1024;
|
||||
|
||||
//Following keys have no defaults
|
||||
public static final String DFS_DATANODE_DATA_DIR_KEY = "dfs.datanode.data.dir";
|
||||
|
|
|
|||
|
|
@ -364,6 +364,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
|||
|
||||
private final long maxFsObjects; // maximum number of fs objects
|
||||
|
||||
private final long minBlockSize; // minimum block size
|
||||
private final long maxBlocksPerFile; // maximum # of blocks per file
|
||||
|
||||
/**
|
||||
* The global generation stamp for this file system.
|
||||
*/
|
||||
|
|
@ -595,6 +598,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
|||
this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY,
|
||||
DFS_NAMENODE_MAX_OBJECTS_DEFAULT);
|
||||
|
||||
this.minBlockSize = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY,
|
||||
DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT);
|
||||
this.maxBlocksPerFile = conf.getLong(DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY,
|
||||
DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT);
|
||||
this.accessTimePrecision = conf.getLong(DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
|
||||
DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT);
|
||||
this.supportAppends = conf.getBoolean(DFS_SUPPORT_APPEND_KEY, DFS_SUPPORT_APPEND_DEFAULT);
|
||||
|
|
@ -1818,6 +1825,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
|||
final HdfsFileStatus stat;
|
||||
FSPermissionChecker pc = getPermissionChecker();
|
||||
checkOperation(OperationCategory.WRITE);
|
||||
if (blockSize < minBlockSize) {
|
||||
throw new IOException("Specified block size is less than configured" +
|
||||
" minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
|
||||
+ "): " + blockSize + " < " + minBlockSize);
|
||||
}
|
||||
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
||||
writeLock();
|
||||
try {
|
||||
|
|
@ -2245,7 +2257,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
|||
// This is a retry. Just return the last block.
|
||||
return onRetryBlock[0];
|
||||
}
|
||||
|
||||
if (pendingFile.getBlocks().length >= maxBlocksPerFile) {
|
||||
throw new IOException("File has reached the limit on maximum number of"
|
||||
+ " blocks (" + DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY
|
||||
+ "): " + pendingFile.getBlocks().length + " >= "
|
||||
+ maxBlocksPerFile);
|
||||
}
|
||||
blockSize = pendingFile.getPreferredBlockSize();
|
||||
clientNode = pendingFile.getClientNode();
|
||||
replication = pendingFile.getBlockReplication();
|
||||
|
|
|
|||
|
|
@ -238,6 +238,23 @@
|
|||
contain. A value of 0 will disable the check.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.fs-limits.min-block-size</name>
|
||||
<value>1048576</value>
|
||||
<description>Minimum block size in bytes, enforced by the Namenode at create
|
||||
time. This prevents the accidental creation of files with tiny block
|
||||
sizes (and thus many blocks), which can degrade
|
||||
performance.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.fs-limits.max-blocks-per-file</name>
|
||||
<value>1048576</value>
|
||||
<description>Maximum number of blocks per file, enforced by the Namenode on
|
||||
write. This prevents the creation of extremely large files which can
|
||||
degrade performance.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>dfs.namenode.edits.dir</name>
|
||||
<value>${dfs.namenode.name.dir}</value>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
|
||||
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
|
||||
import org.apache.hadoop.ipc.RemoteException;
|
||||
import org.apache.hadoop.test.GenericTestUtils;
|
||||
import org.junit.Test;
|
||||
|
||||
|
||||
|
|
@ -159,4 +162,59 @@ public class TestFileLimit {
|
|||
testFileLimit();
|
||||
simulatedStorage = false;
|
||||
}
|
||||
|
||||
@Test(timeout=60000)
|
||||
public void testMaxBlocksPerFileLimit() throws Exception {
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
// Make a small block size and a low limit
|
||||
final long blockSize = 4096;
|
||||
final long numBlocks = 2;
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||
conf.setLong(DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY, numBlocks);
|
||||
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
|
||||
FileSystem fs = cluster.getFileSystem();
|
||||
HdfsDataOutputStream fout =
|
||||
(HdfsDataOutputStream)fs.create(new Path("/testmaxfilelimit"));
|
||||
try {
|
||||
// Write maximum number of blocks
|
||||
fout.write(new byte[(int)blockSize*(int)numBlocks]);
|
||||
fout.hflush();
|
||||
// Try to write one more block
|
||||
try {
|
||||
fout.write(new byte[1]);
|
||||
fout.hflush();
|
||||
assert false : "Expected IOException after writing too many blocks";
|
||||
} catch (IOException e) {
|
||||
GenericTestUtils.assertExceptionContains("File has reached the limit" +
|
||||
" on maximum number of", e);
|
||||
}
|
||||
} finally {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test(timeout=60000)
|
||||
public void testMinBlockSizeLimit() throws Exception {
|
||||
final long blockSize = 4096;
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, blockSize);
|
||||
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
|
||||
FileSystem fs = cluster.getFileSystem();
|
||||
|
||||
try {
|
||||
// Try with min block size
|
||||
fs.create(new Path("/testmblock1"), true, 4096, (short)3, blockSize);
|
||||
try {
|
||||
// Try with min block size - 1
|
||||
fs.create(new Path("/testmblock2"), true, 4096, (short)3, blockSize-1);
|
||||
assert false : "Expected IOException after creating a file with small" +
|
||||
" blocks ";
|
||||
} catch (IOException e) {
|
||||
GenericTestUtils.assertExceptionContains("Specified block size is less",
|
||||
e);
|
||||
}
|
||||
} finally {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,5 +25,10 @@
|
|||
<name>hadoop.security.authentication</name>
|
||||
<value>simple</value>
|
||||
</property>
|
||||
<!-- Disable min block size since most tests use tiny blocks -->
|
||||
<property>
|
||||
<name>dfs.namenode.fs-limits.min-block-size</name>
|
||||
<value>0</value>
|
||||
</property>
|
||||
|
||||
</configuration>
|
||||
|
|
|
|||
Loading…
Reference in New Issue