diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index ca57d307baa..de91f8cc039 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -89,11 +89,6 @@
test-compile
-
-
- mrapp-generated-classpath
-
-
maven-antrun-plugin
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index 7d863a56e42..05e9a9bf76e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -18,13 +18,8 @@
package org.apache.hadoop.mapreduce.v2.util;
-import java.io.BufferedReader;
-import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
import java.net.URI;
-import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -134,62 +129,24 @@ public class MRApps extends Apps {
private static void setMRFrameworkClasspath(
Map environment, Configuration conf) throws IOException {
- InputStream classpathFileStream = null;
- BufferedReader reader = null;
- try {
- // Get yarn mapreduce-app classpath from generated classpath
- // Works if compile time env is same as runtime. Mainly tests.
- ClassLoader thisClassLoader =
- Thread.currentThread().getContextClassLoader();
- String mrAppGeneratedClasspathFile = "mrapp-generated-classpath";
- classpathFileStream =
- thisClassLoader.getResourceAsStream(mrAppGeneratedClasspathFile);
+ // Propagate the system classpath when using the mini cluster
+ if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
+ System.getProperty("java.class.path"));
+ }
- // Put the file itself on classpath for tasks.
- URL classpathResource = thisClassLoader
- .getResource(mrAppGeneratedClasspathFile);
- if (classpathResource != null) {
- String classpathElement = classpathResource.getFile();
- if (classpathElement.contains("!")) {
- classpathElement = classpathElement.substring(0,
- classpathElement.indexOf("!"));
- } else {
- classpathElement = new File(classpathElement).getParent();
- }
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
- classpathElement);
- }
-
- if (classpathFileStream != null) {
- reader = new BufferedReader(new InputStreamReader(classpathFileStream,
- Charsets.UTF_8));
- String cp = reader.readLine();
- if (cp != null) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
- cp.trim());
- }
- }
-
- // Add standard Hadoop classes
- for (String c : conf.getStrings(
- YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
- .trim());
- }
- for (String c : conf.getStrings(
- MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
- MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
- .trim());
- }
- } finally {
- if (classpathFileStream != null) {
- classpathFileStream.close();
- }
- if (reader != null) {
- reader.close();
- }
+ // Add standard Hadoop classes
+ for (String c : conf.getStrings(
+ YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
+ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
+ .trim());
+ }
+ for (String c : conf.getStrings(
+ MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
+ MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
+ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
+ .trim());
}
// TODO: Remove duplicates.
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
index d6a17cf97bd..6f630aecb6b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -52,6 +53,8 @@ public class TestEncryptedShuffle {
private static final String BASEDIR =
System.getProperty("test.build.dir", "target/test-dir") + "/" +
TestEncryptedShuffle.class.getSimpleName();
+
+ private String classpathDir;
@BeforeClass
public static void setUp() throws Exception {
@@ -62,27 +65,12 @@ public class TestEncryptedShuffle {
@Before
public void createCustomYarnClasspath() throws Exception {
- String classpathDir =
- KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
-
- URL url = Thread.currentThread().getContextClassLoader().
- getResource("mrapp-generated-classpath");
- File f = new File(url.getPath());
- BufferedReader reader = new BufferedReader(new FileReader(f));
- String cp = reader.readLine();
- cp = cp + ":" + classpathDir;
- f = new File(classpathDir, "mrapp-generated-classpath");
- Writer writer = new FileWriter(f);
- writer.write(cp);
- writer.close();
+ classpathDir = KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
new File(classpathDir, "core-site.xml").delete();
}
@After
public void cleanUpMiniClusterSpecialConfig() throws Exception {
- String classpathDir =
- KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
- new File(classpathDir, "mrapp-generated-classpath").delete();
new File(classpathDir, "core-site.xml").delete();
String keystoresDir = new File(BASEDIR).getAbsolutePath();
KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, classpathDir);
@@ -98,6 +86,9 @@ public class TestEncryptedShuffle {
conf.set("dfs.block.access.token.enable", "false");
conf.set("dfs.permissions", "true");
conf.set("hadoop.security.authentication", "simple");
+ String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) +
+ File.pathSeparator + classpathDir;
+ conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp);
dfsCluster = new MiniDFSCluster(conf, 1, true, null);
FileSystem fileSystem = dfsCluster.getFileSystem();
fileSystem.mkdirs(new Path("/tmp"));
@@ -113,8 +104,6 @@ public class TestEncryptedShuffle {
mrCluster = MiniMRClientClusterFactory.create(this.getClass(), 1, conf);
// so the minicluster conf is avail to the containers.
- String classpathDir =
- KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
Writer writer = new FileWriter(classpathDir + "/core-site.xml");
mrCluster.getConfig().writeXml(writer);
writer.close();
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 1e7f18575c3..7c7aa73cba4 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -705,11 +705,6 @@
org.apache.maven.plugins
maven-jar-plugin
2.3.1
-
-
- mrapp-generated-classpath
-
-
org.apache.maven.plugins
@@ -804,21 +799,6 @@
-
- maven-dependency-plugin
-
-
- build-classpath
- generate-sources
-
- build-classpath
-
-
- target/classes/mrapp-generated-classpath
-
-
-
-
org.apache.maven.plugins
maven-enforcer-plugin
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e32e30eada8..dd4cd385adf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -54,6 +54,8 @@ Release 2.0.3-alpha - Unreleased
YARN-183. Clean up fair scheduler code. (Sandy Ryza via tomwhite)
+ YARN-129. Simplify classpath construction for mini YARN tests. (tomwhite)
+
OPTIMIZATIONS
BUG FIXES
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 2cc0fd152bc..462de0de1e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -90,23 +90,6 @@
-
- maven-dependency-plugin
-
-
- build-classpath
- generate-sources
-
- build-classpath
-
-
-
- target/classes/yarn-apps-ds-generated-classpath
-
-
-
-
org.apache.maven.plugins
maven-surefire-plugin
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 4a11cd02fcc..d857558fbd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -494,9 +494,10 @@ public class Client extends YarnClientImpl {
classPathEnv.append(":./log4j.properties");
// add the runtime classpath needed for tests to work
- String testRuntimeClassPath = Client.getTestRuntimeClasspath();
- classPathEnv.append(':');
- classPathEnv.append(testRuntimeClassPath);
+ if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+ classPathEnv.append(':');
+ classPathEnv.append(System.getProperty("java.class.path"));
+ }
env.put("CLASSPATH", classPathEnv.toString());
@@ -663,50 +664,4 @@ public class Client extends YarnClientImpl {
super.killApplication(appId);
}
- private static String getTestRuntimeClasspath() {
-
- InputStream classpathFileStream = null;
- BufferedReader reader = null;
- String envClassPath = "";
-
- LOG.info("Trying to generate classpath for app master from current thread's classpath");
- try {
-
- // Create classpath from generated classpath
- // Check maven ppom.xml for generated classpath info
- // Works if compile time env is same as runtime. Mainly tests.
- ClassLoader thisClassLoader =
- Thread.currentThread().getContextClassLoader();
- String generatedClasspathFile = "yarn-apps-ds-generated-classpath";
- classpathFileStream =
- thisClassLoader.getResourceAsStream(generatedClasspathFile);
- if (classpathFileStream == null) {
- LOG.info("Could not classpath resource from class loader");
- return envClassPath;
- }
- LOG.info("Readable bytes from stream=" + classpathFileStream.available());
- reader = new BufferedReader(new InputStreamReader(classpathFileStream));
- String cp = reader.readLine();
- if (cp != null) {
- envClassPath += cp.trim() + ":";
- }
- // Put the file itself on classpath for tasks.
- envClassPath += thisClassLoader.getResource(generatedClasspathFile).getFile();
- } catch (IOException e) {
- LOG.info("Could not find the necessary resource to generate class path for tests. Error=" + e.getMessage());
- }
-
- try {
- if (classpathFileStream != null) {
- classpathFileStream.close();
- }
- if (reader != null) {
- reader.close();
- }
- } catch (IOException e) {
- LOG.info("Failed to close class path file stream or reader. Error=" + e.getMessage());
- }
- return envClassPath;
- }
-
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
index 2c5113b9f3e..8508a209030 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
@@ -86,23 +86,6 @@
-
- maven-dependency-plugin
-
-
- build-classpath
- generate-sources
-
- build-classpath
-
-
-
- target/classes/yarn-apps-am-generated-classpath
-
-
-
-
org.apache.maven.plugins
maven-surefire-plugin
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
index 199f64d36eb..6bb863e9f07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
@@ -18,12 +18,9 @@
package org.apache.hadoop.yarn.applications.unmanagedamlauncher;
-import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.URL;
@@ -80,51 +77,17 @@ public class TestUnmanagedAMLauncher {
}
private static String getTestRuntimeClasspath() {
-
- InputStream classpathFileStream = null;
- BufferedReader reader = null;
- String envClassPath = "";
-
LOG.info("Trying to generate classpath for app master from current thread's classpath");
- try {
-
- // Create classpath from generated classpath
- // Check maven pom.xml for generated classpath info
- // Works if compile time env is same as runtime. Mainly tests.
- ClassLoader thisClassLoader = Thread.currentThread()
- .getContextClassLoader();
- String generatedClasspathFile = "yarn-apps-am-generated-classpath";
- classpathFileStream = thisClassLoader
- .getResourceAsStream(generatedClasspathFile);
- if (classpathFileStream == null) {
- LOG.info("Could not classpath resource from class loader");
- return envClassPath;
- }
- LOG.info("Readable bytes from stream=" + classpathFileStream.available());
- reader = new BufferedReader(new InputStreamReader(classpathFileStream));
- String cp = reader.readLine();
- if (cp != null) {
- envClassPath += cp.trim() + File.pathSeparator;
- }
- // yarn-site.xml at this location contains proper config for mini cluster
- URL url = thisClassLoader.getResource("yarn-site.xml");
- envClassPath += new File(url.getFile()).getParent();
- } catch (IOException e) {
- LOG.info("Could not find the necessary resource to generate class path for tests. Error="
- + e.getMessage());
- }
-
- try {
- if (classpathFileStream != null) {
- classpathFileStream.close();
- }
- if (reader != null) {
- reader.close();
- }
- } catch (IOException e) {
- LOG.info("Failed to close class path file stream or reader. Error="
- + e.getMessage());
+ String envClassPath = "";
+ String cp = System.getProperty("java.class.path");
+ if (cp != null) {
+ envClassPath += cp.trim() + File.pathSeparator;
}
+ // yarn-site.xml at this location contains proper config for mini cluster
+ ClassLoader thisClassLoader = Thread.currentThread()
+ .getContextClassLoader();
+ URL url = thisClassLoader.getResource("yarn-site.xml");
+ envClassPath += new File(url.getFile()).getParent();
return envClassPath;
}