HDFS-3796. Speed up edit log tests by avoiding fsync(). Contributed by Todd Lipcon.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373567 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
9dff7876d9
commit
89153a9c68
|
@ -397,6 +397,8 @@ Branch-2 ( Unreleased changes )
|
|||
HDFS-3802. StartupOption.name in HdfsServerConstants should be final.
|
||||
(Jing Zhao via szetszwo)
|
||||
|
||||
HDFS-3796. Speed up edit log tests by avoiding fsync() (todd)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HDFS-2982. Startup performance suffers when there are many edit log
|
||||
|
|
|
@ -49,6 +49,8 @@ public class EditLogFileOutputStream extends EditLogOutputStream {
|
|||
private EditsDoubleBuffer doubleBuf;
|
||||
static ByteBuffer fill = ByteBuffer.allocateDirect(MIN_PREALLOCATION_LENGTH);
|
||||
|
||||
private static boolean shouldSkipFsyncForTests = false;
|
||||
|
||||
static {
|
||||
fill.position(0);
|
||||
for (int i = 0; i < fill.capacity(); i++) {
|
||||
|
@ -184,7 +186,9 @@ public class EditLogFileOutputStream extends EditLogOutputStream {
|
|||
}
|
||||
preallocate(); // preallocate file if necessay
|
||||
doubleBuf.flushTo(fp);
|
||||
fc.force(false); // metadata updates not needed
|
||||
if (!shouldSkipFsyncForTests) {
|
||||
fc.force(false); // metadata updates not needed
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -247,4 +251,15 @@ public class EditLogFileOutputStream extends EditLogOutputStream {
|
|||
public FileChannel getFileChannelForTesting() {
|
||||
return fc;
|
||||
}
|
||||
|
||||
/**
|
||||
* For the purposes of unit tests, we don't need to actually
|
||||
* write durably to disk. So, we can skip the fsync() calls
|
||||
* for a speed improvement.
|
||||
* @param skip true if fsync should <em>not</em> be called
|
||||
*/
|
||||
@VisibleForTesting
|
||||
public static void setShouldSkipFsyncForTesting(boolean skip) {
|
||||
shouldSkipFsyncForTests = skip;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -119,6 +119,11 @@ public class TestEditLog {
|
|||
"a4ff 0000 0000 0000 0000 0000 0000 0000"
|
||||
).replace(" ",""));
|
||||
|
||||
static {
|
||||
// No need to fsync for the purposes of tests. This makes
|
||||
// the tests run much faster.
|
||||
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
|
||||
}
|
||||
|
||||
static final byte TRAILER_BYTE = FSEditLogOpCodes.OP_INVALID.getOpCode();
|
||||
|
||||
|
|
|
@ -40,6 +40,12 @@ public class TestEditLogFileOutputStream {
|
|||
final static int MIN_PREALLOCATION_LENGTH =
|
||||
EditLogFileOutputStream.MIN_PREALLOCATION_LENGTH;
|
||||
|
||||
static {
|
||||
// No need to fsync for the purposes of tests. This makes
|
||||
// the tests run much faster.
|
||||
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
|
||||
}
|
||||
|
||||
@Before
|
||||
@After
|
||||
public void deleteEditsFile() {
|
||||
|
|
|
@ -51,6 +51,12 @@ import com.google.common.collect.ImmutableList;
|
|||
public class TestFileJournalManager {
|
||||
static final Log LOG = LogFactory.getLog(TestFileJournalManager.class);
|
||||
|
||||
static {
|
||||
// No need to fsync for the purposes of tests. This makes
|
||||
// the tests run much faster.
|
||||
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find out how many transactions we can read from a
|
||||
* FileJournalManager, starting at a given transaction ID.
|
||||
|
|
|
@ -57,6 +57,7 @@ public class TestNameNodeRecovery {
|
|||
|
||||
static {
|
||||
recoverStartOpt.setForce(MetaRecoveryContext.FORCE_ALL);
|
||||
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
|
||||
}
|
||||
|
||||
static void runEditLogTest(EditLogTestSetup elts) throws IOException {
|
||||
|
|
|
@ -49,6 +49,12 @@ public class TestSecurityTokenEditLog {
|
|||
static final int NUM_THREADS = 100;
|
||||
static final int opsPerTrans = 3;
|
||||
|
||||
static {
|
||||
// No need to fsync for the purposes of tests. This makes
|
||||
// the tests run much faster.
|
||||
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
|
||||
}
|
||||
|
||||
//
|
||||
// an object that does a bunch of transactions
|
||||
//
|
||||
|
|
|
@ -34,6 +34,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
|
|||
import org.apache.hadoop.hdfs.HAUtil;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
|
||||
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
|
||||
|
@ -52,6 +53,12 @@ public class TestEditLogsDuringFailover {
|
|||
private static final Log LOG =
|
||||
LogFactory.getLog(TestEditLogsDuringFailover.class);
|
||||
private static final int NUM_DIRS_IN_LOG = 5;
|
||||
|
||||
static {
|
||||
// No need to fsync for the purposes of tests. This makes
|
||||
// the tests run much faster.
|
||||
EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStartup() throws Exception {
|
||||
|
|
Loading…
Reference in New Issue