YARN-129. Simplify classpath construction for mini YARN tests.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1411235 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml

@@ -89,11 +89,6 @@
             <phase>test-compile</phase>
           </execution>
         </executions>
-        <configuration>
-          <excludes>
-            <exclude>mrapp-generated-classpath</exclude>
-          </excludes>
-        </configuration>
       </plugin>
       <plugin>
         <artifactId>maven-antrun-plugin</artifactId>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java

@@ -18,13 +18,8 @@
 
 package org.apache.hadoop.mapreduce.v2.util;
 
-import java.io.BufferedReader;
-import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.net.URI;
-import java.net.URL;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -134,62 +129,24 @@ public static TaskAttemptStateUI taskAttemptState(String attemptStateStr) {
 
   private static void setMRFrameworkClasspath(
       Map<String, String> environment, Configuration conf) throws IOException {
-    InputStream classpathFileStream = null;
-    BufferedReader reader = null;
-    try {
-      // Get yarn mapreduce-app classpath from generated classpath
-      // Works if compile time env is same as runtime. Mainly tests.
-      ClassLoader thisClassLoader =
-          Thread.currentThread().getContextClassLoader();
-      String mrAppGeneratedClasspathFile = "mrapp-generated-classpath";
-      classpathFileStream =
-          thisClassLoader.getResourceAsStream(mrAppGeneratedClasspathFile);
+    // Propagate the system classpath when using the mini cluster
+    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+          System.getProperty("java.class.path"));
+    }
 
-      // Put the file itself on classpath for tasks.
-      URL classpathResource = thisClassLoader
-        .getResource(mrAppGeneratedClasspathFile);
-      if (classpathResource != null) {
-        String classpathElement = classpathResource.getFile();
-        if (classpathElement.contains("!")) {
-          classpathElement = classpathElement.substring(0,
-            classpathElement.indexOf("!"));
-        } else {
-          classpathElement = new File(classpathElement).getParent();
-        }
-        Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
-          classpathElement);
-      }
-
-      if (classpathFileStream != null) {
-        reader = new BufferedReader(new InputStreamReader(classpathFileStream, 
-            Charsets.UTF_8));
-        String cp = reader.readLine();
-        if (cp != null) {
-          Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
-            cp.trim());
-        }
-      }
-
-      // Add standard Hadoop classes
-      for (String c : conf.getStrings(
-          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
-          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
-        Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
-            .trim());
-      }
-      for (String c : conf.getStrings(
-          MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
-          MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
-        Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
-            .trim());
-      }
-    } finally {
-      if (classpathFileStream != null) {
-        classpathFileStream.close();
-      }
-      if (reader != null) {
-        reader.close();
-      }
+    // Add standard Hadoop classes
+    for (String c : conf.getStrings(
+        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
+          .trim());
+    }
+    for (String c : conf.getStrings(
+        MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
+        MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
+      Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
+          .trim());
     }
     // TODO: Remove duplicates.
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java

@@ -31,6 +31,7 @@
 
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -52,6 +53,8 @@ public class TestEncryptedShuffle {
   private static final String BASEDIR =
     System.getProperty("test.build.dir", "target/test-dir") + "/" +
     TestEncryptedShuffle.class.getSimpleName();
+  
+  private String classpathDir;
 
   @BeforeClass
   public static void setUp() throws Exception {
@@ -62,27 +65,12 @@ public static void setUp() throws Exception {
 
   @Before
   public void createCustomYarnClasspath() throws Exception {
-    String classpathDir =
-      KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
-
-    URL url = Thread.currentThread().getContextClassLoader().
-      getResource("mrapp-generated-classpath");
-    File f = new File(url.getPath());
-    BufferedReader reader = new BufferedReader(new FileReader(f));
-    String cp = reader.readLine();
-    cp = cp + ":" + classpathDir;
-    f = new File(classpathDir, "mrapp-generated-classpath");
-    Writer writer = new FileWriter(f);
-    writer.write(cp);
-    writer.close();
+    classpathDir = KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
     new File(classpathDir, "core-site.xml").delete();
   }
 
   @After
   public void cleanUpMiniClusterSpecialConfig() throws Exception {
-    String classpathDir =
-      KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
-    new File(classpathDir, "mrapp-generated-classpath").delete();
     new File(classpathDir, "core-site.xml").delete();
     String keystoresDir = new File(BASEDIR).getAbsolutePath();
     KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, classpathDir);
@@ -98,6 +86,9 @@ private void startCluster(Configuration  conf) throws Exception {
     conf.set("dfs.block.access.token.enable", "false");
     conf.set("dfs.permissions", "true");
     conf.set("hadoop.security.authentication", "simple");
+    String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) +
+      File.pathSeparator + classpathDir;
+    conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp);
     dfsCluster = new MiniDFSCluster(conf, 1, true, null);
     FileSystem fileSystem = dfsCluster.getFileSystem();
     fileSystem.mkdirs(new Path("/tmp"));
@@ -113,8 +104,6 @@ private void startCluster(Configuration  conf) throws Exception {
     mrCluster = MiniMRClientClusterFactory.create(this.getClass(), 1, conf);
 
     // so the minicluster conf is avail to the containers.
-    String classpathDir =
-      KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
     Writer writer = new FileWriter(classpathDir + "/core-site.xml");
     mrCluster.getConfig().writeXml(writer);
     writer.close();

hadoop-project/pom.xml

@@ -703,11 +703,6 @@
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-jar-plugin</artifactId>
           <version>2.3.1</version>
-          <configuration>
-            <excludes>
-              <exclude>mrapp-generated-classpath</exclude>
-            </excludes>
-          </configuration>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
@@ -802,21 +797,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>build-classpath</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>build-classpath</goal>
-            </goals>
-            <configuration>
-              <outputFile>target/classes/mrapp-generated-classpath</outputFile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>

hadoop-yarn-project/CHANGES.txt

@@ -71,6 +71,8 @@ Release 2.0.3-alpha - Unreleased
 
     YARN-183. Clean up fair scheduler code. (Sandy Ryza via tomwhite)
 
+    YARN-129. Simplify classpath construction for mini YARN tests. (tomwhite)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml

@@ -90,23 +90,6 @@
            </archive>
         </configuration>
       </plugin>
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>build-classpath</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>build-classpath</goal>
-            </goals>
-            <configuration>
-              <!-- needed to run the unit test for DS to generate the required classpath 
-                   that is required in the env of the launch container in the mini yarn cluster -->
-              <outputFile>target/classes/yarn-apps-ds-generated-classpath</outputFile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java

@@ -494,9 +494,10 @@ else if (amMemory > maxMem) {
     classPathEnv.append(":./log4j.properties");
 
     // add the runtime classpath needed for tests to work
-    String testRuntimeClassPath = Client.getTestRuntimeClasspath();
-    classPathEnv.append(':');
-    classPathEnv.append(testRuntimeClassPath);
+    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+      classPathEnv.append(':');
+      classPathEnv.append(System.getProperty("java.class.path"));
+    }
 
     env.put("CLASSPATH", classPathEnv.toString());
 
@@ -663,50 +664,4 @@ private void forceKillApplication(ApplicationId appId) throws YarnRemoteExceptio
     super.killApplication(appId);	
   }
 
-  private static String getTestRuntimeClasspath() {
-
-    InputStream classpathFileStream = null;
-    BufferedReader reader = null;
-    String envClassPath = "";
-
-    LOG.info("Trying to generate classpath for app master from current thread's classpath");
-    try {
-
-      // Create classpath from generated classpath
-      // Check maven ppom.xml for generated classpath info
-      // Works if compile time env is same as runtime. Mainly tests.
-      ClassLoader thisClassLoader =
-          Thread.currentThread().getContextClassLoader();
-      String generatedClasspathFile = "yarn-apps-ds-generated-classpath";
-      classpathFileStream =
-          thisClassLoader.getResourceAsStream(generatedClasspathFile);
-      if (classpathFileStream == null) {
-        LOG.info("Could not classpath resource from class loader");
-        return envClassPath;
-      }
-      LOG.info("Readable bytes from stream=" + classpathFileStream.available());
-      reader = new BufferedReader(new InputStreamReader(classpathFileStream));
-      String cp = reader.readLine();
-      if (cp != null) {
-        envClassPath += cp.trim() + ":";
-      }
-      // Put the file itself on classpath for tasks.
-      envClassPath += thisClassLoader.getResource(generatedClasspathFile).getFile();
-    } catch (IOException e) {
-      LOG.info("Could not find the necessary resource to generate class path for tests. Error=" + e.getMessage());
-    } 
-
-    try {
-      if (classpathFileStream != null) {
-        classpathFileStream.close();
-      }
-      if (reader != null) {
-        reader.close();
-      }
-    } catch (IOException e) {
-      LOG.info("Failed to close class path file stream or reader. Error=" + e.getMessage());
-    } 
-    return envClassPath;
-  }			
-
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml

@@ -86,23 +86,6 @@
 
   <build>
     <plugins>
-      <plugin>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>build-classpath</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>build-classpath</goal>
-            </goals>
-            <configuration>
-              <!-- needed to run the unit test for DS to generate the required classpath 
-                   that is required in the env of the launch container in the mini yarn cluster -->
-              <outputFile>target/classes/yarn-apps-am-generated-classpath</outputFile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java

@@ -18,12 +18,9 @@
 
 package org.apache.hadoop.yarn.applications.unmanagedamlauncher;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.URL;
 
@@ -80,51 +77,17 @@ public static void tearDown() throws IOException {
   }
 
   private static String getTestRuntimeClasspath() {
-
-    InputStream classpathFileStream = null;
-    BufferedReader reader = null;
-    String envClassPath = "";
-
     LOG.info("Trying to generate classpath for app master from current thread's classpath");
-    try {
-
-      // Create classpath from generated classpath
-      // Check maven pom.xml for generated classpath info
-      // Works if compile time env is same as runtime. Mainly tests.
-      ClassLoader thisClassLoader = Thread.currentThread()
-          .getContextClassLoader();
-      String generatedClasspathFile = "yarn-apps-am-generated-classpath";
-      classpathFileStream = thisClassLoader
-          .getResourceAsStream(generatedClasspathFile);
-      if (classpathFileStream == null) {
-        LOG.info("Could not classpath resource from class loader");
-        return envClassPath;
-      }
-      LOG.info("Readable bytes from stream=" + classpathFileStream.available());
-      reader = new BufferedReader(new InputStreamReader(classpathFileStream));
-      String cp = reader.readLine();
-      if (cp != null) {
-        envClassPath += cp.trim() + File.pathSeparator;
-      }
-      // yarn-site.xml at this location contains proper config for mini cluster
-      URL url = thisClassLoader.getResource("yarn-site.xml");
-      envClassPath += new File(url.getFile()).getParent();
-    } catch (IOException e) {
-      LOG.info("Could not find the necessary resource to generate class path for tests. Error="
-          + e.getMessage());
-    }
-
-    try {
-      if (classpathFileStream != null) {
-        classpathFileStream.close();
-      }
-      if (reader != null) {
-        reader.close();
-      }
-    } catch (IOException e) {
-      LOG.info("Failed to close class path file stream or reader. Error="
-          + e.getMessage());
+    String envClassPath = "";
+    String cp = System.getProperty("java.class.path");
+    if (cp != null) {
+      envClassPath += cp.trim() + File.pathSeparator;
     }
+    // yarn-site.xml at this location contains proper config for mini cluster
+    ClassLoader thisClassLoader = Thread.currentThread()
+      .getContextClassLoader();
+    URL url = thisClassLoader.getResource("yarn-site.xml");
+    envClassPath += new File(url.getFile()).getParent();
     return envClassPath;
   }