Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-6773. MiniDFSCluster should skip edit log fsync by default. Contributed by Stephen Chu.

This commit is contained in:
Colin Patrick Mccabe 2014-08-27 13:33:02 -07:00
parent fdd3bc5f45
commit d805cc27a9
4 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -518,6 +518,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6690. Deduplicate xattr names in memory. (wang)
HDFS-6773. MiniDFSCluster should skip edit log fsync by default (Stephen
Chu via Colin Patrick McCabe)
BUG FIXES
HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for

22
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -93,6 +93,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@ -172,6 +173,7 @@ public class MiniDFSCluster {
private boolean checkDataNodeAddrConfig = false;
private boolean checkDataNodeHostConfig = false;
private Configuration[] dnConfOverlays;
private boolean skipFsyncForTesting = true;
public Builder(Configuration conf) {
this.conf = conf;
@ -406,6 +408,15 @@ public class MiniDFSCluster {
return this;
}
/**
* Default: true
* When true, we skip fsync() calls for speed improvements.
*/
public Builder skipFsyncForTesting(boolean val) {
this.skipFsyncForTesting = val;
return this;
}
/**
* Construct the actual MiniDFSCluster
*/
@ -472,7 +483,8 @@ public class MiniDFSCluster {
builder.checkExitOnShutdown,
builder.checkDataNodeAddrConfig,
builder.checkDataNodeHostConfig,
builder.dnConfOverlays);
builder.dnConfOverlays,
builder.skipFsyncForTesting);
}
public class DataNodeProperties {
@ -727,7 +739,8 @@ public class MiniDFSCluster {
manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs,
operation, null, racks, hosts,
null, simulatedCapacities, null, true, false,
MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false, null);
MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0),
true, false, false, null, true);
}
private void initMiniDFSCluster(
@ -742,7 +755,8 @@ public class MiniDFSCluster {
MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown,
boolean checkDataNodeAddrConfig,
boolean checkDataNodeHostConfig,
Configuration[] dnConfOverlays)
Configuration[] dnConfOverlays,
boolean skipFsyncForTesting)
throws IOException {
boolean success = false;
try {
@ -783,6 +797,8 @@ public class MiniDFSCluster {
conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1);
}
EditLogFileOutputStream.setShouldSkipFsyncForTesting(skipFsyncForTesting);
federation = nnTopology.isFederated();
try {
createNameNodesAndSetConf(

1
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
View File

@ -108,7 +108,6 @@ public class TestFsDatasetCache {
private static CacheManipulator prevCacheManipulator;
static {
EditLogFileOutputStream.setShouldSkipFsyncForTesting(false);
LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG);
}

1
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
View File

@ -110,7 +110,6 @@ public class TestCacheDirectives {
static {
NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator());
EditLogFileOutputStream.setShouldSkipFsyncForTesting(false);
}
private static final long BLOCK_SIZE = 4096;