HDFS-8356. Document missing properties in hdfs-default.xml. Contributed by Ray Chiang.

hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md

@@ -33,6 +33,7 @@ The following table lists the configuration property names that are deprecated i
 | dfs.http.address | dfs.namenode.http-address |
 | dfs.https.address | dfs.namenode.https-address |
 | dfs.https.client.keystore.resource | dfs.client.https.keystore.resource |
+| dfs.https.enable | dfs.http.policy |
 | dfs.https.need.client.auth | dfs.client.https.need-auth |
 | dfs.max.objects | dfs.namenode.max.objects |
 | dfs.max-repl-streams | dfs.namenode.replication.max-streams |
@@ -51,6 +52,7 @@ The following table lists the configuration property names that are deprecated i
 | dfs.secondary.http.address | dfs.namenode.secondary.http-address |
 | dfs.socket.timeout | dfs.client.socket-timeout |
 | dfs.umaskmode | fs.permissions.umask-mode |
+| dfs.web.ugi | hadoop.http.staticuser.user |
 | dfs.write.packet.size | dfs.client-write-packet-size |
 | fs.checkpoint.dir | dfs.namenode.checkpoint.dir |
 | fs.checkpoint.edits.dir | dfs.namenode.checkpoint.edits.dir |

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java

@@ -41,11 +41,74 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
   public void initializeMemberVariables() {
     xmlFilename = new String("hdfs-default.xml");
     configurationClasses = new Class[] { HdfsClientConfigKeys.class,
-        DFSConfigKeys.class};
+        DFSConfigKeys.class };
 
     // Set error modes
     errorIfMissingConfigProps = true;
-    errorIfMissingXmlProps = false;
+    errorIfMissingXmlProps = true;
+
+    // Initialize used variables
+    configurationPropsToSkipCompare = new HashSet<String>();
+
+    // Ignore testing based parameter
+    configurationPropsToSkipCompare.add("ignore.secure.ports.for.testing");
+
+    // Remove deprecated properties listed in Configuration#DeprecationDelta
+    configurationPropsToSkipCompare.add(DFSConfigKeys.DFS_DF_INTERVAL_KEY);
+
+    // Remove deprecated properties in DeprecatedProperties.md
+    configurationPropsToSkipCompare.add(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY);
+
+    // Remove default properties
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_DEFAULT);
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_WEBHDFS_AUTHENTICATION_FILTER_DEFAULT);
+
+    // Remove support property
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY);
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_DATANODE_MIN_SUPPORTED_NAMENODE_VERSION_KEY);
+
+    // Purposely hidden, based on comments in DFSConfigKeys
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY);
+
+    // Fully deprecated properties?
+    configurationPropsToSkipCompare
+        .add("dfs.corruptfilesreturned.max");
+    configurationPropsToSkipCompare
+        .add("dfs.datanode.hdfs-blocks-metadata.enabled");
+    configurationPropsToSkipCompare
+        .add("dfs.metrics.session-id");
+    configurationPropsToSkipCompare
+        .add("dfs.datanode.synconclose");
+    configurationPropsToSkipCompare
+        .add("dfs.datanode.non.local.lazy.persist");
+    configurationPropsToSkipCompare
+        .add("dfs.namenode.tolerate.heartbeat.multiplier");
+    configurationPropsToSkipCompare
+        .add("dfs.namenode.stripe.min");
+    configurationPropsToSkipCompare
+        .add("dfs.namenode.replqueue.threshold-pct");
+
+    // Removed by HDFS-6440
+    configurationPropsToSkipCompare
+        .add("dfs.ha.log-roll.rpc.timeout");
+
+    // Example (not real) property in hdfs-default.xml
+    configurationPropsToSkipCompare.add("dfs.ha.namenodes");
+
+    // Property used for internal testing only
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION);
+
+    // Property not intended for users
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY);
+    configurationPropsToSkipCompare
+        .add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
 
     // Allocate
     xmlPropsToSkipCompare = new HashSet<String>();
@@ -58,29 +121,36 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
     // Used dynamically as part of DFSConfigKeys.DFS_NAMENODE_EDITS_PLUGIN_PREFIX
     xmlPropsToSkipCompare.add("dfs.namenode.edits.journal-plugin.qjournal");
 
-    // Example (not real) property in hdfs-default.xml
-    xmlPropsToSkipCompare.add("dfs.ha.namenodes.EXAMPLENAMESERVICE");
-
     // Defined in org.apache.hadoop.fs.CommonConfigurationKeys
     xmlPropsToSkipCompare.add("hadoop.user.group.metrics.percentiles.intervals");
 
     // Used oddly by DataNode to create new config String
     xmlPropsToSkipCompare.add("hadoop.hdfs.configuration.version");
 
-    // Kept in the NfsConfiguration class in the hadoop-hdfs-nfs module
-    xmlPrefixToSkipCompare.add("nfs");
-
-    // Not a hardcoded property.  Used by SaslRpcClient
-    xmlPrefixToSkipCompare.add("dfs.namenode.kerberos.principal.pattern");
-
-    // Skip comparing in branch-2.  Removed in trunk with HDFS-7985.
-    xmlPropsToSkipCompare.add("dfs.webhdfs.enabled");
-
     // Some properties have moved to HdfsClientConfigKeys
     xmlPropsToSkipCompare.add("dfs.client.short.circuit.replica.stale.threshold.ms");
 
     // Ignore HTrace properties
     xmlPropsToSkipCompare.add("fs.client.htrace");
     xmlPropsToSkipCompare.add("hadoop.htrace");
+
+    // Ignore SpanReceiveHost properties
+    xmlPropsToSkipCompare.add("dfs.htrace.spanreceiver.classes");
+    xmlPropsToSkipCompare.add("dfs.client.htrace.spanreceiver.classes");
+
+    // Remove deprecated properties listed in Configuration#DeprecationDelta
+    xmlPropsToSkipCompare.add(DFSConfigKeys.DFS_DF_INTERVAL_KEY);
+
+    // Skip this until TestConfigurationFieldsBase can integrate DfsClientConf
+    xmlPropsToSkipCompare.add("dfs.datanode.hdfs-blocks-metadata.enabled");
+
+    // Kept in the NfsConfiguration class in the hadoop-hdfs-nfs module
+    xmlPrefixToSkipCompare.add("nfs");
+
+    // Not a hardcoded property.  Used by SaslRpcClient
+    xmlPrefixToSkipCompare.add("dfs.namenode.kerberos.principal.pattern");
+
+    // Skip over example property
+    xmlPrefixToSkipCompare.add("dfs.ha.namenodes");
   }
 }