Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-2759. Pre-allocate HDFS edit log files after writing version number. Contributed by Aaron T. Myers. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1338489 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2012-05-15 00:31:35 +00:00
parent b81b987be8
commit 201038272f
3 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -548,6 +548,9 @@ Release 2.0.0 - UNRELEASED
HDFS-3031. Fix complete() and getAdditionalBlock() RPCs to be idempotent HDFS-3031. Fix complete() and getAdditionalBlock() RPCs to be idempotent
(todd) (todd)
HDFS-2759. Pre-allocate HDFS edit log files after writing version number.
(atm)
BREAKDOWN OF HDFS-1623 SUBTASKS BREAKDOWN OF HDFS-1623 SUBTASKS
HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd) HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd)

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
View File

@ -182,10 +182,10 @@ public class EditLogFileOutputStream extends EditLogOutputStream {
LOG.info("Nothing to flush"); LOG.info("Nothing to flush");
return; return;
} }
preallocate(); // preallocate file if necessary
doubleBuf.flushTo(fp); doubleBuf.flushTo(fp);
fc.force(false); // metadata updates not needed because of preallocation fc.force(false); // metadata updates not needed
fc.position(fc.position() - 1); // skip back the end-of-file marker fc.position(fc.position() - 1); // skip back the end-of-file marker
preallocate(); // preallocate file if necessary
} }
/** /**

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
View File

@ -41,6 +41,7 @@ import org.mockito.Mockito;
public class TestEditLogFileOutputStream { public class TestEditLogFileOutputStream {
private final static long PREALLOCATION_LENGTH = (1024 * 1024) + 4;
private final static int HEADER_LEN = 17; private final static int HEADER_LEN = 17;
private static final File TEST_EDITS = private static final File TEST_EDITS =
new File(System.getProperty("test.build.data","/tmp"), new File(System.getProperty("test.build.data","/tmp"),
@ -65,8 +66,9 @@ public class TestEditLogFileOutputStream {
assertEquals("Edit log should contain a header as valid length", assertEquals("Edit log should contain a header as valid length",
HEADER_LEN, validation.getValidLength()); HEADER_LEN, validation.getValidLength());
assertEquals(1, validation.getNumTransactions()); assertEquals(1, validation.getNumTransactions());
assertEquals("Edit log should have 1MB of bytes allocated", assertEquals("Edit log should have 1MB pre-allocated, plus 4 bytes " +
1024*1024, editLog.length()); "for the version number",
PREALLOCATION_LENGTH, editLog.length());
cluster.getFileSystem().mkdirs(new Path("/tmp"), cluster.getFileSystem().mkdirs(new Path("/tmp"),
@ -79,8 +81,8 @@ public class TestEditLogFileOutputStream {
validation.getValidLength() > oldLength); validation.getValidLength() > oldLength);
assertEquals(2, validation.getNumTransactions()); assertEquals(2, validation.getNumTransactions());
assertEquals("Edit log should be 1MB long", assertEquals("Edit log should be 1MB long, plus 4 bytes for the version number",
1024 * 1024, editLog.length()); PREALLOCATION_LENGTH, editLog.length());
// 256 blocks for the 1MB of preallocation space // 256 blocks for the 1MB of preallocation space
assertTrue("Edit log disk space used should be at least 257 blocks", assertTrue("Edit log disk space used should be at least 257 blocks",
256 * 4096 <= new DU(editLog, conf).getUsed()); 256 * 4096 <= new DU(editLog, conf).getUsed());