diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 29654cf39f6..0f12bd30823 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -166,6 +166,9 @@ Release 2.6.0 - UNRELEASED HDFS-6690. Deduplicate xattr names in memory. (wang) + HDFS-6773. MiniDFSCluster should skip edit log fsync by default (Stephen + Chu via Colin Patrick McCabe) + BUG FIXES HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 2cc783540e6..e7577bed6c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -94,6 +94,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -174,6 +175,7 @@ public class MiniDFSCluster { private boolean checkDataNodeAddrConfig = false; private boolean checkDataNodeHostConfig = false; private Configuration[] dnConfOverlays; + private boolean skipFsyncForTesting = true; public Builder(Configuration conf) { this.conf = conf; @@ -407,6 +409,15 @@ public class MiniDFSCluster { this.dnConfOverlays = dnConfOverlays; return this; } + + /** + * Default: true + * When true, we skip fsync() calls for speed improvements. + */ + public Builder skipFsyncForTesting(boolean val) { + this.skipFsyncForTesting = val; + return this; + } /** * Construct the actual MiniDFSCluster @@ -474,7 +485,8 @@ public class MiniDFSCluster { builder.checkExitOnShutdown, builder.checkDataNodeAddrConfig, builder.checkDataNodeHostConfig, - builder.dnConfOverlays); + builder.dnConfOverlays, + builder.skipFsyncForTesting); } public class DataNodeProperties { @@ -727,7 +739,8 @@ public class MiniDFSCluster { manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, operation, null, racks, hosts, null, simulatedCapacities, null, true, false, - MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false, null); + MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), + true, false, false, null, true); } private void initMiniDFSCluster( @@ -742,7 +755,8 @@ public class MiniDFSCluster { MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig, - Configuration[] dnConfOverlays) + Configuration[] dnConfOverlays, + boolean skipFsyncForTesting) throws IOException { boolean success = false; try { @@ -785,6 +799,8 @@ public class MiniDFSCluster { + "Standby node since no IPC ports have been specified."); conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1); } + + EditLogFileOutputStream.setShouldSkipFsyncForTesting(skipFsyncForTesting); federation = nnTopology.isFederated(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java index 5ac13eec270..d6e70d80037 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java @@ -108,7 +108,6 @@ public class TestFsDatasetCache { private static CacheManipulator prevCacheManipulator; static { - EditLogFileOutputStream.setShouldSkipFsyncForTesting(false); LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index d54b90e6631..93076928f3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -110,7 +110,6 @@ public class TestCacheDirectives { static { NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); - EditLogFileOutputStream.setShouldSkipFsyncForTesting(false); } private static final long BLOCK_SIZE = 4096;