From 7226a71b1f684f562bd88ee121f1dd7aa8b73816 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 30 Nov 2016 15:58:31 -0800 Subject: [PATCH] HDFS-5517. Lower the default maximum number of blocks per file. Contributed by Aaron T. Myers and Andrew Wang. --- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +- .../hdfs/server/datanode/TestDirectoryScanner.java | 11 +++++++++-- .../server/namenode/metrics/TestNameNodeMetrics.java | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index d7d3c9dd6ba..df218579aa1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -399,7 +399,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_MIN_BLOCK_SIZE_KEY = "dfs.namenode.fs-limits.min-block-size"; public static final long DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT = 1024*1024; public static final String DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY = "dfs.namenode.fs-limits.max-blocks-per-file"; - public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 1024*1024; + public static final long DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT = 10*1000; public static final String DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY = "dfs.namenode.fs-limits.max-xattrs-per-inode"; public static final int DFS_NAMENODE_MAX_XATTRS_PER_INODE_DEFAULT = 32; public static final String DFS_NAMENODE_MAX_XATTR_SIZE_KEY = "dfs.namenode.fs-limits.max-xattr-size"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 671c98c3b95..086f6673b53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -372,7 +372,7 @@ dfs.namenode.fs-limits.max-blocks-per-file - 1048576 + 10000 Maximum number of blocks per file, enforced by the Namenode on write. This prevents the creation of extremely large files which can degrade performance. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index f08b579d1f3..d7c8383d7a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -590,8 +590,15 @@ public class TestDirectoryScanner { 100); DataNode dataNode = cluster.getDataNodes().get(0); - createFile(GenericTestUtils.getMethodName(), - BLOCK_LENGTH * blocks, false); + final int maxBlocksPerFile = (int) DFSConfigKeys + .DFS_NAMENODE_MAX_BLOCKS_PER_FILE_DEFAULT; + int numBlocksToCreate = blocks; + while (numBlocksToCreate > 0) { + final int toCreate = Math.min(maxBlocksPerFile, numBlocksToCreate); + createFile(GenericTestUtils.getMethodName() + numBlocksToCreate, + BLOCK_LENGTH * toCreate, false); + numBlocksToCreate -= toCreate; + } float ratio = 0.0f; int retries = maxRetries; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index 0ad613090c3..b9d25b22584 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -571,7 +571,7 @@ public class TestNameNodeMetrics { Path file1_Path = new Path(TEST_ROOT_DIR_PATH, "ReadData.dat"); //Perform create file operation - createFile(file1_Path, 1024 * 1024,(short)2); + createFile(file1_Path, 1024, (short) 2); // Perform read file operation on earlier created file readFile(fs, file1_Path);