Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12603. Enable async edit logging by default. Contributed by Andrew Wang.

This commit is contained in:
Xiao Chen 2017-10-16 09:43:39 -07:00
parent 9fcc3a1fc8
commit 035c6ee587
4 changed files with 21 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING = public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging"; "dfs.namenode.edits.asynclogging";
public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false; public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
public static final String DFS_LIST_LIMIT = "dfs.ls.limit"; public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000; public static final int DFS_LIST_LIMIT_DEFAULT = 1000;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
View File

@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
* *
* In a non-HA setup: * In a non-HA setup:
* *
* The log starts in UNITIALIZED state upon construction. Once it's * The log starts in UNINITIALIZED state upon construction. Once it's
* initialized, it is usually in IN_SEGMENT state, indicating that edits may * initialized, it is usually in IN_SEGMENT state, indicating that edits may
* be written. In the middle of a roll, or while saving the namespace, it * be written. In the middle of a roll, or while saving the namespace, it
* briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous * briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous

2
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -4105,7 +4105,7 @@
<property> <property>
<name>dfs.namenode.edits.asynclogging</name> <name>dfs.namenode.edits.asynclogging</name>
<value>false</value> <value>true</value>
<description> <description>
If set to true, enables asynchronous edit logs in the Namenode. If set If set to true, enables asynchronous edit logs in the Namenode. If set
to false, the Namenode uses the traditional synchronous edit logs. to false, the Namenode uses the traditional synchronous edit logs.

21
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
View File

@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
private static final Random RANDOM = new Random(); private static final Random RANDOM = new Random();
private final TestType clusterType; private final TestType clusterType;
private final boolean useAsyncEditLogging;
private Configuration conf; private Configuration conf;
private MiniDFSCluster cluster; private MiniDFSCluster cluster;
private MiniQJMHACluster miniQjmHaCluster; // for QJM case only private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@ -88,18 +89,26 @@ public class TestFailureToReadEdits {
}; };
/** /**
* Run this suite of tests both for QJM-based HA and for file-based * Run this suite of tests for {QJM-based, file-based HA} x {async
* HA. * edit logging enabled, disabled}.
*
* TODO: Enable the test cases with async edit logging on. See HDFS-12603
* and HDFS-12660.
*/ */
@Parameters @Parameters
public static Iterable<Object[]> data() { public static Iterable<Object[]> data() {
return Arrays.asList(new Object[][]{ return Arrays.asList(new Object[][]{
{ TestType.SHARED_DIR_HA }, {TestType.SHARED_DIR_HA, Boolean.FALSE},
{ TestType.QJM_HA } }); //{TestType.SHARED_DIR_HA, Boolean.TRUE},
{TestType.QJM_HA, Boolean.FALSE},
//{TestType.QJM_HA, Boolean.TRUE},
});
} }
public TestFailureToReadEdits(TestType clusterType) { public TestFailureToReadEdits(TestType clusterType, Boolean
useAsyncEditLogging) {
this.clusterType = clusterType; this.clusterType = clusterType;
this.useAsyncEditLogging = useAsyncEditLogging;
} }
@Before @Before
@ -109,6 +118,8 @@ public class TestFailureToReadEdits {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10); conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
useAsyncEditLogging);
HAUtil.setAllowStandbyReads(conf, true); HAUtil.setAllowStandbyReads(conf, true);
if (clusterType == TestType.SHARED_DIR_HA) { if (clusterType == TestType.SHARED_DIR_HA) {