Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12603. Enable async edit logging by default. Contributed by Andrew Wang. 

(cherry picked from commit d0311dfc5fe92a0e0f3fb8ae92e412edce609740)
This commit is contained in:
Xiao Chen 2017-10-16 09:43:39 -07:00
parent 66847c27db
commit 77ef810552
4 changed files with 21 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -323,7 +323,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging";
public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
View File

@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
*
* In a non-HA setup:
*
* The log starts in UNITIALIZED state upon construction. Once it's
* The log starts in UNINITIALIZED state upon construction. Once it's
* initialized, it is usually in IN_SEGMENT state, indicating that edits may
* be written. In the middle of a roll, or while saving the namespace, it
* briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous

2
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -4105,7 +4105,7 @@
<property>
<name>dfs.namenode.edits.asynclogging</name>
<value>false</value>
<value>true</value>
<description>
If set to true, enables asynchronous edit logs in the Namenode. If set
to false, the Namenode uses the traditional synchronous edit logs.

25
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
View File

@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
private static final Random RANDOM = new Random();
private final TestType clusterType;
private final boolean useAsyncEditLogging;
private Configuration conf;
private MiniDFSCluster cluster;
private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@ -88,18 +89,26 @@ public class TestFailureToReadEdits {
};
/**
* Run this suite of tests both for QJM-based HA and for file-based
* HA.
* Run this suite of tests for {QJM-based, file-based HA} x {async
* edit logging enabled, disabled}.
*
* TODO: Enable the test cases with async edit logging on. See HDFS-12603
* and HDFS-12660.
*/
@Parameters
public static Iterable<Object[]> data() {
return Arrays.asList(new Object[][] {
{ TestType.SHARED_DIR_HA },
{ TestType.QJM_HA } });
return Arrays.asList(new Object[][]{
{TestType.SHARED_DIR_HA, Boolean.FALSE},
//{TestType.SHARED_DIR_HA, Boolean.TRUE},
{TestType.QJM_HA, Boolean.FALSE},
//{TestType.QJM_HA, Boolean.TRUE},
});
}
public TestFailureToReadEdits(TestType clusterType) {
public TestFailureToReadEdits(TestType clusterType, Boolean
useAsyncEditLogging) {
this.clusterType = clusterType;
this.useAsyncEditLogging = useAsyncEditLogging;
}
@Before
@ -109,6 +118,8 @@ public class TestFailureToReadEdits {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
useAsyncEditLogging);
HAUtil.setAllowStandbyReads(conf, true);
if (clusterType == TestType.SHARED_DIR_HA) {