MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter)

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java

@@ -146,6 +146,12 @@ public abstract class TestConfigurationFieldsBase {
    */
   private Set<String> xmlFieldsMissingInConfiguration = null;
 
+  /**
+   * Member variable for debugging base class operation
+   */
+  protected boolean configDebug = false;
+  protected boolean xmlDebug = false;
+
   /**
    * Abstract method to be used by subclasses for initializing base
    * members.
@@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase {
     HashMap<String,String> retVal = new HashMap<String,String>();
 
     // Setup regexp for valid properties
-    String propRegex = "^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$";
+    String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
     Pattern p = Pattern.compile(propRegex);
 
     // Iterate through class member variables
     int totalFields = 0;
     String value;
     for (Field f : fields) {
+      if (configDebug) {
+        System.out.println("Field: " + f);
+      }
       // Filter out anything that isn't "public static final"
       if (!Modifier.isStatic(f.getModifiers()) ||
           !Modifier.isPublic(f.getModifiers()) ||
@@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase {
       } catch (IllegalAccessException iaException) {
         continue;
       }
+      if (configDebug) {
+        System.out.println("  Value: " + value);
+      }
       // Special Case: Detect and ignore partial properties (ending in x)
       //               or file properties (ending in .xml)
       if (value.endsWith(".xml") ||
@@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase {
       //                  something like: blah.blah2(.blah3.blah4...)
       Matcher m = p.matcher(value);
       if (!m.find()) {
+        if (configDebug) {
+          System.out.println("  Passes Regex: false");
+        }
         continue;
       }
+      if (configDebug) {
+        System.out.println("  Passes Regex: true");
+      }
 
       // Save member variable/value as hash
-      retVal.put(value,f.getName());
+      if (!retVal.containsKey(value)) {
+        retVal.put(value,f.getName());
+      } else {
+        if (configDebug) {
+          System.out.println("ERROR: Already found key for property " + value);
+        }
+      }
     }
 
     return retVal;
@@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase {
       // Ignore known xml props
       if (xmlPropsToSkipCompare != null) {
         if (xmlPropsToSkipCompare.contains(key)) {
+          if (xmlDebug) {
+            System.out.println("  Skipping Full Key: " + key);
+          }
           continue;
         }
       }
@@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase {
 	}
       }
       if (skipPrefix) {
+        if (xmlDebug) {
+          System.out.println("  Skipping Prefix Key: " + key);
+        }
         continue;
       }
       if (conf.onlyKeyExists(key)) {
         retVal.put(key,null);
+        if (xmlDebug) {
+          System.out.println("  XML Key,Null Value: " + key);
+        }
       } else {
         String value = conf.get(key);
         if (value!=null) {
           retVal.put(key,entry.getValue());
+          if (xmlDebug) {
+            System.out.println("  XML Key,Valid Value: " + key);
+          }
         }
       }
       kvItr.remove();
@@ -312,6 +348,10 @@ public abstract class TestConfigurationFieldsBase {
 
     // Create class member/value map
     configurationMemberVariables = new HashMap<String,String>();
+    if (configDebug) {
+      System.out.println("Reading configuration classes");
+      System.out.println("");
+    }
     for (Class c : configurationClasses) {
       Field[] fields = c.getDeclaredFields();
       Map<String,String> memberMap =
@@ -320,9 +360,23 @@ public abstract class TestConfigurationFieldsBase {
         configurationMemberVariables.putAll(memberMap);
       }
     }
+    if (configDebug) {
+      System.out.println("");
+      System.out.println("=====");
+      System.out.println("");
+    }
 
     // Create XML key/value map
+    if (xmlDebug) {
+      System.out.println("Reading XML property files");
+      System.out.println("");
+    }
     xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
+    if (xmlDebug) {
+      System.out.println("");
+      System.out.println("=====");
+      System.out.println("");
+    }
 
     // Find class members not in the XML file
     configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields

hadoop-mapreduce-project/CHANGES.txt

@@ -304,6 +304,9 @@ Release 2.8.0 - UNRELEASED
     mapreduce.tasktracker.taskmemorymanager.monitoringinterval.
     (J.Andreina via aajisaka)
 
+    MAPREDUCE-6192. Create unit test to automatically compare
+    MR related classes and mapred-default.xml (rchiang via rkanter)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java

@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+import java.util.HashSet;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+
+/**
+ * Unit test class to compare the following MR Configuration classes:
+ * <p></p>
+ * {@link org.apache.hadoop.mapreduce.MRJobConfig}
+ * {@link org.apache.hadoop.mapreduce.MRConfig}
+ * {@link org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig}
+ * {@link org.apache.hadoop.mapred.ShuffleHandler}
+ * {@link org.apache.hadoop.mapreduce.lib.output.FileOutputFormat}
+ * {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}
+ * {@link org.apache.hadoop.mapreduce.Job}
+ * {@link org.apache.hadoop.mapreduce.lib.input.NLineInputFormat}
+ * {@link org.apache.hadoop.mapred.JobConf}
+ * <p></p>
+ * against mapred-default.xml for missing properties.  Currently only
+ * throws an error if the class is missing a property.
+ * <p></p>
+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
+ * for how this class works.
+ */
+public class TestMapreduceConfigFields extends TestConfigurationFieldsBase {
+
+  @SuppressWarnings("deprecation")
+  @Override
+  public void initializeMemberVariables() {
+    xmlFilename = new String("mapred-default.xml");
+    configurationClasses = new Class[] { MRJobConfig.class, MRConfig.class,
+        JHAdminConfig.class, ShuffleHandler.class, FileOutputFormat.class,
+	FileInputFormat.class, Job.class, NLineInputFormat.class,
+	JobConf.class, FileOutputCommitter.class };
+
+    // Initialize used variables
+    configurationPropsToSkipCompare = new HashSet<String>();
+
+    // Set error modes
+    errorIfMissingConfigProps = true;
+    errorIfMissingXmlProps = false;
+
+    // Ignore deprecated MR1 properties in JobConf
+    configurationPropsToSkipCompare
+            .add(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY);
+    configurationPropsToSkipCompare
+            .add(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
+  }
+
+}