From 9809a16d3c8068beccbf0106e99c7ede6ba11e0f Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Mon, 4 May 2015 17:48:10 -0700 Subject: [PATCH] MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter) --- .../conf/TestConfigurationFieldsBase.java | 58 +++++++++++++- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapred/TestMapreduceConfigFields.java | 76 +++++++++++++++++++ 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java index c3fe3a39f5b..2e4d8b1c1ba 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java @@ -146,6 +146,12 @@ public abstract class TestConfigurationFieldsBase { */ private Set xmlFieldsMissingInConfiguration = null; + /** + * Member variable for debugging base class operation + */ + protected boolean configDebug = false; + protected boolean xmlDebug = false; + /** * Abstract method to be used by subclasses for initializing base * members. @@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase { HashMap retVal = new HashMap(); // Setup regexp for valid properties - String propRegex = "^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$"; + String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$"; Pattern p = Pattern.compile(propRegex); // Iterate through class member variables int totalFields = 0; String value; for (Field f : fields) { + if (configDebug) { + System.out.println("Field: " + f); + } // Filter out anything that isn't "public static final" if (!Modifier.isStatic(f.getModifiers()) || !Modifier.isPublic(f.getModifiers()) || @@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase { } catch (IllegalAccessException iaException) { continue; } + if (configDebug) { + System.out.println(" Value: " + value); + } // Special Case: Detect and ignore partial properties (ending in x) // or file properties (ending in .xml) if (value.endsWith(".xml") || @@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase { // something like: blah.blah2(.blah3.blah4...) Matcher m = p.matcher(value); if (!m.find()) { + if (configDebug) { + System.out.println(" Passes Regex: false"); + } continue; } + if (configDebug) { + System.out.println(" Passes Regex: true"); + } // Save member variable/value as hash - retVal.put(value,f.getName()); + if (!retVal.containsKey(value)) { + retVal.put(value,f.getName()); + } else { + if (configDebug) { + System.out.println("ERROR: Already found key for property " + value); + } + } } return retVal; @@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase { // Ignore known xml props if (xmlPropsToSkipCompare != null) { if (xmlPropsToSkipCompare.contains(key)) { + if (xmlDebug) { + System.out.println(" Skipping Full Key: " + key); + } continue; } } @@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase { } } if (skipPrefix) { + if (xmlDebug) { + System.out.println(" Skipping Prefix Key: " + key); + } continue; } if (conf.onlyKeyExists(key)) { retVal.put(key,null); + if (xmlDebug) { + System.out.println(" XML Key,Null Value: " + key); + } } else { String value = conf.get(key); if (value!=null) { retVal.put(key,entry.getValue()); + if (xmlDebug) { + System.out.println(" XML Key,Valid Value: " + key); + } } } kvItr.remove(); @@ -312,6 +348,10 @@ public abstract class TestConfigurationFieldsBase { // Create class member/value map configurationMemberVariables = new HashMap(); + if (configDebug) { + System.out.println("Reading configuration classes"); + System.out.println(""); + } for (Class c : configurationClasses) { Field[] fields = c.getDeclaredFields(); Map memberMap = @@ -320,9 +360,23 @@ public abstract class TestConfigurationFieldsBase { configurationMemberVariables.putAll(memberMap); } } + if (configDebug) { + System.out.println(""); + System.out.println("====="); + System.out.println(""); + } // Create XML key/value map + if (xmlDebug) { + System.out.println("Reading XML property files"); + System.out.println(""); + } xmlKeyValueMap = extractPropertiesFromXml(xmlFilename); + if (xmlDebug) { + System.out.println(""); + System.out.println("====="); + System.out.println(""); + } // Find class members not in the XML file configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 002fbe6abac..99621cb384b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -304,6 +304,9 @@ Release 2.8.0 - UNRELEASED mapreduce.tasktracker.taskmemorymanager.monitoringinterval. (J.Andreina via aajisaka) + MAPREDUCE-6192. Create unit test to automatically compare + MR related classes and mapred-default.xml (rchiang via rkanter) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java new file mode 100644 index 00000000000..7f187147c01 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce; + +import java.util.HashSet; + +import org.apache.hadoop.conf.TestConfigurationFieldsBase; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.ShuffleHandler; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; + +/** + * Unit test class to compare the following MR Configuration classes: + *

+ * {@link org.apache.hadoop.mapreduce.MRJobConfig} + * {@link org.apache.hadoop.mapreduce.MRConfig} + * {@link org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig} + * {@link org.apache.hadoop.mapred.ShuffleHandler} + * {@link org.apache.hadoop.mapreduce.lib.output.FileOutputFormat} + * {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat} + * {@link org.apache.hadoop.mapreduce.Job} + * {@link org.apache.hadoop.mapreduce.lib.input.NLineInputFormat} + * {@link org.apache.hadoop.mapred.JobConf} + *

+ * against mapred-default.xml for missing properties. Currently only + * throws an error if the class is missing a property. + *

+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase} + * for how this class works. + */ +public class TestMapreduceConfigFields extends TestConfigurationFieldsBase { + + @SuppressWarnings("deprecation") + @Override + public void initializeMemberVariables() { + xmlFilename = new String("mapred-default.xml"); + configurationClasses = new Class[] { MRJobConfig.class, MRConfig.class, + JHAdminConfig.class, ShuffleHandler.class, FileOutputFormat.class, + FileInputFormat.class, Job.class, NLineInputFormat.class, + JobConf.class, FileOutputCommitter.class }; + + // Initialize used variables + configurationPropsToSkipCompare = new HashSet(); + + // Set error modes + errorIfMissingConfigProps = true; + errorIfMissingXmlProps = false; + + // Ignore deprecated MR1 properties in JobConf + configurationPropsToSkipCompare + .add(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY); + configurationPropsToSkipCompare + .add(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY); + } + +}