HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be order-independent. (Yitong Zhou via gera)

(cherry picked from commit 0bcea111e5)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -199,6 +199,9 @@ Release 2.7.0 - UNRELEASED
 
     HADOOP-11386. Replace \n by %n in format hadoop-common format strings.
     (Li Lu via wheat9)
+
+    HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be
+    be order-independent. (Yitong Zhou via gera)
     
 Release 2.6.0 - 2014-11-18
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java

@@ -216,28 +216,43 @@ public class ApplicationClassLoader extends URLClassLoader {
     return c;
   }
 
+  /**
+   * Checks if a class should be included as a system class.
+   *
+   * A class is a system class if and only if it matches one of the positive
+   * patterns and none of the negative ones.
+   *
+   * @param name the class name to check
+   * @param systemClasses a list of system class configurations.
+   * @return true if the class is a system class
+   */
   public static boolean isSystemClass(String name, List<String> systemClasses) {
+    boolean result = false;
     if (systemClasses != null) {
       String canonicalName = name.replace('/', '.');
       while (canonicalName.startsWith(".")) {
         canonicalName=canonicalName.substring(1);
       }
       for (String c : systemClasses) {
-        boolean result = true;
+        boolean shouldInclude = true;
         if (c.startsWith("-")) {
           c = c.substring(1);
-          result = false;
+          shouldInclude = false;
         }
         if (canonicalName.startsWith(c)) {
           if (   c.endsWith(".")                                   // package
               || canonicalName.length() == c.length()              // class
               ||    canonicalName.length() > c.length()            // nested
                  && canonicalName.charAt(c.length()) == '$' ) {
-            return result;
+            if (shouldInclude) {
+              result = true;
+            } else {
+              return false;
+            }
           }
         }
       }
     }
-    return false;
+    return result;
   }
-}
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java

@@ -87,7 +87,7 @@ public class TestApplicationClassLoader {
     assertEquals(jarFile.toURI().toURL(), urls[2]);
     // nofile should be ignored
   }
-  
+
   @Test
   public void testIsSystemClass() {
     testIsSystemClassInternal("");
@@ -112,8 +112,12 @@ public class TestApplicationClassLoader {
         classes("-org.example.Foo,org.example.")));
     assertTrue(isSystemClass("org.example.Bar" + nestedClass,
         classes("-org.example.Foo.,org.example.")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.,-org.example.Foo")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.Foo,-org.example.Foo")));
   }
-  
+
   private List<String> classes(String classes) {
     return Lists.newArrayList(Splitter.on(',').split(classes));
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml

@@ -1798,10 +1798,23 @@
    <value></value>
   <description>Used to override the default definition of the system classes for
     the job classloader. The system classes are a comma-separated list of
-    classes that should be loaded from the system classpath, not the
+    patterns that indicate whether to load a class from the system classpath,
-    user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending
+    instead from the user-supplied JARs, when mapreduce.job.classloader is
-    in '.' (period) are treated as package names, and names starting with a '-'
+    enabled.
-    are treated as negative matches.
+
+    A positive pattern is defined as:
+        1. A single class name 'C' that matches 'C' and transitively all nested
+            classes 'C$*' defined in C;
+        2. A package name ending with a '.' (e.g., "com.example.") that matches
+            all classes from that package.
+    A negative pattern is defined by a '-' in front of a positive pattern
+    (e.g., "-com.example.").
+
+    A class is considered a system class if and only if it matches one of the
+    positive patterns and none of the negative ones. More formally:
+    A class is a member of the inclusion set I if it matches one of the positive
+    patterns. A class is a member of the exclusion set E if it matches one of
+    the negative patterns. The set of system classes S = I \ E.
   </description>
 </property>