From 2d01ffaf24fb511f589ae594549d6eca452c07fd Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Wed, 4 May 2016 15:12:51 +0900 Subject: [PATCH] HADOOP-12101. Add automatic search of default Configuration variables to TestConfigurationFieldsBase. Contributed by Ray Chiang. (cherry picked from commit 355325bcc7111fa4aac801fd23a26422ffabaf7c) --- dev-support/verify-xml.sh | 150 +++++++++++ .../conf/TestConfigurationFieldsBase.java | 255 ++++++++++++++++++ 2 files changed, 405 insertions(+) create mode 100755 dev-support/verify-xml.sh diff --git a/dev-support/verify-xml.sh b/dev-support/verify-xml.sh new file mode 100755 index 00000000000..abab4e69f2b --- /dev/null +++ b/dev-support/verify-xml.sh @@ -0,0 +1,150 @@ +#!/bin/bash +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +## +# Script to run unit tests for xml <-> 1 or more Configuration file verification +# usage: ./verify-xml.sh +# + +# Utility functions +function find_test_output_file() { + echo "Found test output file(s) at" + echo "" + if [ -n "$1" ] && [ -e "$1" ] ; then + echo " $1" + fi + if [ -n "$2" ] && [ -e "$2" ] ; then + echo " $2" + fi + if [ -n "$3" ] && [ -e "$3" ] ; then + echo " $3" + fi + if [ -n "$4" ] && [ -e "$4" ] ; then + echo " $4" + fi + echo "" + echo "Examine the file for specific information xml/Configuration mismatches." + echo "" +} + +function print_test_banner() { + local banner_text=$1 + local banner_length=${#banner_text} + local banner + banner=$( printf "%${banner_length}s" ' ' ) + echo "" + echo "${banner// /=}" + echo "${banner_text}" + echo "${banner// /=}" + echo "" +} + +# Wrapper functions for running unit tests +function run_all_xml_test() { + mvn test -Dtest=TestCommonConfigurationFields,TestHdfsConfigFields,TestMapreduceConfigFields,TestYarnConfigurationFields + if [ $? -ne 0 ] ; then + print_test_banner "All Test*ConfigFields FAIL" + else + print_test_banner "All Test*ConfigFields SUCCESS" + fi +} + +function run_common_xml_test() { + mvn test -Dtest=TestCommonConfigFields + if [ $? -ne 0 ] ; then + print_test_banner "TestCommonConfigurationFields FAIL" + else + print_test_banner "TestCommonConfigurationFields SUCCESS" + fi +} + +function run_hdfs_xml_test() { + mvn test -Dtest=TestHdfsConfigFields + if [ $? -ne 0 ] ; then + print_test_banner "TestHdfsConfigFields FAIL" + else + print_test_banner "TestHdfsConfigFields SUCCESS" + fi +} + +function run_mapreduce_xml_test() { + mvn test -Dtest=TestMapreduceConfigFields + if [ $? -ne 0 ] ; then + print_test_banner "TestMapreduceConfigFields FAIL" + else + print_test_banner "TestMapreduceConfigFields SUCCESS" + fi +} + +function run_yarn_xml_test() { + mvn test -Dtest=TestYarnConfigurationFields + if [ $? -ne 0 ] ; then + print_test_banner "TestYarnConfigurationFields FAIL" + else + print_test_banner "TestYarnConfigurationFields SUCCESS" + fi +} + +# Main body +cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")/.." || exit +dir="$(pwd -P)" + +# - Create unit test file names +export commonOutputFile +commonOutputFile="$(find "${dir}" -name org.apache.hadoop.conf.TestCommonConfigurationFields-output.txt)" +export hdfsOutputFile +hdfsOutputFile="$(find "${dir}" -name org.apache.hadoop.tools.TestHdfsConfigFields-output.txt)" +export mrOutputFile +mrOutputFile="$(find "${dir}" -name org.apache.hadoop.mapreduce.TestMapreduceConfigFields-output.txt)" +export yarnOutputFile +yarnOutputFile="$(find "${dir}" -name org.apache.hadoop.yarn.conf.TestYarnConfigurationFields-output.txt)" + +# - Determine which tests to run +case "$1" in + + all) + run_all_xml_test + find_test_output_file "${commonOutputFile}" "${hdfsOutputFile}" "${mrOutputFile}" "${yarnOutputFile}" + ;; + + common) + run_common_xml_test + find_test_output_file "${commonOutputFile}" + ;; + + hdfs) + run_hdfs_xml_test + find_test_output_file "${hdfsOutputFile}" + ;; + + mr) + run_mapreduce_xml_test + find_test_output_file "${mrOutputFile}" + ;; + + yarn) + run_yarn_xml_test + find_test_output_file "${yarnOutputFile}" + ;; + + *) + echo "Usage: $0 " + echo " where is one of all, common, hdfs, mr, yarn" + exit 1 + ;; + +esac diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java index e52860294a0..eab01611a6e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java @@ -124,6 +124,11 @@ public abstract class TestConfigurationFieldsBase { */ private Map configurationMemberVariables = null; + /** + * Member variable to store Configuration variables for later reference. + */ + private Map configurationDefaultVariables = null; + /** * Member variable to store XML properties for later comparison. */ @@ -146,6 +151,7 @@ public abstract class TestConfigurationFieldsBase { */ protected boolean configDebug = false; protected boolean xmlDebug = false; + protected boolean defaultDebug = false; /** * Abstract method to be used by subclasses for initializing base @@ -316,6 +322,79 @@ public abstract class TestConfigurationFieldsBase { return retVal; } + /** + * Utility function to extract "public static final" default + * member variables from a Configuration type class. + * + * @param fields The class member variables + * @return HashMap containing entries + */ + private HashMap + extractDefaultVariablesFromConfigurationFields(Field[] fields) { + // Sanity Check + if (fields==null) { + return null; + } + + HashMap retVal = new HashMap(); + + // Setup regexp for valid properties + String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$"; + Pattern p = Pattern.compile(propRegex); + + // Iterate through class member variables + int totalFields = 0; + String value; + for (Field f : fields) { + // Filter out anything that isn't "public static final" + if (!Modifier.isStatic(f.getModifiers()) || + !Modifier.isPublic(f.getModifiers()) || + !Modifier.isFinal(f.getModifiers())) { + continue; + } + // Special: Stuff any property beginning with "DEFAULT_" into a + // different hash for later processing + if (f.getName().startsWith("DEFAULT_") || + f.getName().endsWith("_DEFAULT")) { + if (retVal.containsKey(f.getName())) { + continue; + } + try { + if (f.getType().getName().equals("java.lang.String")) { + String sValue = (String) f.get(null); + retVal.put(f.getName(),sValue); + } else if (f.getType().getName().equals("short")) { + short shValue = (short) f.get(null); + retVal.put(f.getName(),Integer.toString(shValue)); + } else if (f.getType().getName().equals("int")) { + int iValue = (int) f.get(null); + retVal.put(f.getName(),Integer.toString(iValue)); + } else if (f.getType().getName().equals("long")) { + long lValue = (long) f.get(null); + retVal.put(f.getName(),Long.toString(lValue)); + } else if (f.getType().getName().equals("float")) { + float fValue = (float) f.get(null); + retVal.put(f.getName(),Float.toString(fValue)); + } else if (f.getType().getName().equals("double")) { + double dValue = (double) f.get(null); + retVal.put(f.getName(),Double.toString(dValue)); + } else if (f.getType().getName().equals("boolean")) { + boolean bValue = (boolean) f.get(null); + retVal.put(f.getName(),Boolean.toString(bValue)); + } else { + if (defaultDebug) { + System.out.println("Config variable " + f.getName() + " has unknown type " + f.getType().getName()); + } + } + } catch (IllegalAccessException iaException) { + iaException.printStackTrace(); + } + } + } + + return retVal; + } + /** * Perform set difference operation on keyMap2 from keyMap1. * @@ -374,6 +453,26 @@ public abstract class TestConfigurationFieldsBase { System.out.println(""); } + // Create default configuration variable key/value map + if (defaultDebug) { + System.out.println("Reading Config property files for defaults"); + System.out.println(""); + } + configurationDefaultVariables = new HashMap(); + for (Class c : configurationClasses) { + Field[] fields = c.getDeclaredFields(); + Map defaultMap = + extractDefaultVariablesFromConfigurationFields(fields); + if (defaultMap!=null) { + configurationDefaultVariables.putAll(defaultMap); + } + } + if (defaultDebug) { + System.out.println(""); + System.out.println("====="); + System.out.println(""); + } + // Find class members not in the XML file configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields (configurationMemberVariables, xmlKeyValueMap); @@ -464,4 +563,160 @@ public abstract class TestConfigurationFieldsBase { assertTrue(configErrorMsg.toString(), missingConfigSize==0); } } + + /** + * For each property in the XML file, verify that the value matches + * up to the default if one exists. + */ + @Test + public void testXmlAgainstDefaultValuesInConfigurationClass() { + // Error if subclass hasn't set class members + assertTrue(xmlFilename!=null); + assertTrue(configurationMemberVariables!=null); + assertTrue(configurationDefaultVariables!=null); + + HashSet xmlPropertiesWithEmptyValue = new HashSet(); + HashSet configPropertiesWithNoDefaultConfig = new HashSet(); + HashMap xmlPropertiesMatchingConfigDefault = + new HashMap(); + // Ugly solution. Should have tuple-based solution. + HashMap,HashMap> mismatchingXmlConfig = + new HashMap,HashMap>(); + + for (Map.Entry xEntry : xmlKeyValueMap.entrySet()) { + String xmlProperty = xEntry.getKey(); + String xmlDefaultValue = xEntry.getValue(); + String configProperty = configurationMemberVariables.get(xmlProperty); + if (configProperty!=null) { + String defaultConfigName = null; + String defaultConfigValue = null; + + // Type 1: Prepend DEFAULT_ + String defaultNameCheck1 = "DEFAULT_" + configProperty; + String defaultValueCheck1 = configurationDefaultVariables + .get(defaultNameCheck1); + // Type 2: Swap _KEY suffix with _DEFAULT suffix + String defaultNameCheck2 = null; + if (configProperty.endsWith("_KEY")) { + defaultNameCheck2 = configProperty + .substring(0,configProperty.length()-4) + "_DEFAULT"; + } + String defaultValueCheck2 = configurationDefaultVariables + .get(defaultNameCheck2); + // Type Last: Append _DEFAULT suffix + String defaultNameCheck3 = configProperty + "_DEFAULT"; + String defaultValueCheck3 = configurationDefaultVariables + .get(defaultNameCheck3); + + // Pick the default value that exists + if (defaultValueCheck1!=null) { + defaultConfigName = defaultNameCheck1; + defaultConfigValue = defaultValueCheck1; + } else if (defaultValueCheck2!=null) { + defaultConfigName = defaultNameCheck2; + defaultConfigValue = defaultValueCheck2; + } else if (defaultValueCheck3!=null) { + defaultConfigName = defaultNameCheck3; + defaultConfigValue = defaultValueCheck3; + } + + if (defaultConfigValue!=null) { + if (xmlDefaultValue==null) { + xmlPropertiesWithEmptyValue.add(xmlProperty); + } else if (!xmlDefaultValue.equals(defaultConfigValue)) { + HashMap xmlEntry = + new HashMap(); + xmlEntry.put(xmlProperty,xmlDefaultValue); + HashMap configEntry = + new HashMap(); + configEntry.put(defaultConfigName,defaultConfigValue); + mismatchingXmlConfig.put(xmlEntry,configEntry); + } else { + xmlPropertiesMatchingConfigDefault + .put(xmlProperty, defaultConfigName); + } + } else { + configPropertiesWithNoDefaultConfig.add(configProperty); + } + } else { + } + } + + // Print out any unknown mismatching XML value/Config default value + System.out.println(this.xmlFilename + " has " + + mismatchingXmlConfig.size() + + " properties that do not match the default Config value"); + if (mismatchingXmlConfig.size()==0) { + System.out.println(" (None)"); + } else { + for (Map.Entry,HashMap> xcEntry : + mismatchingXmlConfig.entrySet()) { + HashMap xmlMap = xcEntry.getKey(); + HashMap configMap = xcEntry.getValue(); + for (Map.Entry xmlEntry : xmlMap.entrySet()) { + System.out.println(" XML Property: " + xmlEntry.getKey()); + System.out.println(" XML Value: " + xmlEntry.getValue()); + } + for (Map.Entry configEntry : configMap.entrySet()) { + System.out.println(" Config Name: " + configEntry.getKey()); + System.out.println(" Config Value: " + configEntry.getValue()); + } + System.out.println(""); + } + } + System.out.println(); + + // Print out Config properties that have no corresponding DEFAULT_* + // variable and cannot do any XML comparison (i.e. probably needs to + // be checked by hand) + System.out.println("Configuration(s) have " + + configPropertiesWithNoDefaultConfig.size() + + " properties with no corresponding default member variable. These" + + " will need to be verified manually."); + if (configPropertiesWithNoDefaultConfig.size()==0) { + System.out.println(" (None)"); + } else { + Iterator cItr = configPropertiesWithNoDefaultConfig.iterator(); + while (cItr.hasNext()) { + System.out.println(" " + cItr.next()); + } + } + System.out.println(); + + // MAYBE TODO Print out any known mismatching XML value/Config default + + // Print out XML properties that have empty values (i.e. should result + // in code-based default) + System.out.println(this.xmlFilename + " has " + + xmlPropertiesWithEmptyValue.size() + " properties with empty values"); + if (xmlPropertiesWithEmptyValue.size()==0) { + System.out.println(" (None)"); + } else { + Iterator xItr = xmlPropertiesWithEmptyValue.iterator(); + while (xItr.hasNext()) { + System.out.println(" " + xItr.next()); + } + } + System.out.println(); + + // Print out any matching XML value/Config default value + System.out.println(this.xmlFilename + " has " + + xmlPropertiesMatchingConfigDefault.size() + + " properties which match a corresponding Config variable"); + if (xmlPropertiesMatchingConfigDefault.size()==0) { + System.out.println(" (None)"); + } else { + for (Map.Entry xcEntry : + xmlPropertiesMatchingConfigDefault.entrySet()) { + System.out.println(" " + xcEntry.getKey() + " / " + + xcEntry.getValue()); + } + } + System.out.println(); + + // Test separator + System.out.println(); + System.out.println("====="); + System.out.println(); + } }