From a066134277716a9709b63cfb11b4be5627a9bd3f Mon Sep 17 00:00:00 2001 From: cnauroth Date: Tue, 21 Oct 2014 12:33:21 -0700 Subject: [PATCH] YARN-2720. Windows: Wildcard classpath variables not expanded against resources contained in archives. Contributed by Craig Welch. (cherry picked from commit 6637e3cf95b3a9be8d6b9cd66bc849a0607e8ed5) --- .../java/org/apache/hadoop/fs/FileUtil.java | 17 +++++++++++++---- .../java/org/apache/hadoop/util/Classpath.java | 2 +- .../java/org/apache/hadoop/fs/TestFileUtil.java | 4 +++- hadoop-yarn-project/CHANGES.txt | 3 +++ .../WindowsSecureContainerExecutor.java | 6 ++++-- .../launcher/ContainerLaunch.java | 6 ++++-- 6 files changed, 28 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 2b0529356e6..0848cb1cd4c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -1212,10 +1212,11 @@ public class FileUtil { * @param pwd Path to working directory to save jar * @param callerEnv Map caller's environment variables to use * for expansion - * @return String absolute path to new jar + * @return String[] with absolute path to new jar in position 0 and + * unexpanded wild card entry path in position 1 * @throws IOException if there is an I/O error while writing the jar file */ - public static String createJarWithClassPath(String inputClassPath, Path pwd, + public static String[] createJarWithClassPath(String inputClassPath, Path pwd, Map callerEnv) throws IOException { // Replace environment variables, case-insensitive on Windows @SuppressWarnings("unchecked") @@ -1235,6 +1236,7 @@ public class FileUtil { LOG.debug("mkdirs false for " + workingDir + ", execution will continue"); } + StringBuilder unexpandedWildcardClasspath = new StringBuilder(); // Append all entries List classPathEntryList = new ArrayList( classPathEntries.length); @@ -1243,16 +1245,22 @@ public class FileUtil { continue; } if (classPathEntry.endsWith("*")) { + boolean foundWildCardJar = false; // Append all jars that match the wildcard Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}"); FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util() .globStatus(globPath); if (wildcardJars != null) { for (FileStatus wildcardJar: wildcardJars) { + foundWildCardJar = true; classPathEntryList.add(wildcardJar.getPath().toUri().toURL() .toExternalForm()); } } + if (!foundWildCardJar) { + unexpandedWildcardClasspath.append(File.pathSeparator); + unexpandedWildcardClasspath.append(classPathEntry); + } } else { // Append just this entry File fileCpEntry = null; @@ -1300,7 +1308,8 @@ public class FileUtil { } finally { IOUtils.cleanup(LOG, jos, bos, fos); } - - return classPathJar.getCanonicalPath(); + String[] jarCp = {classPathJar.getCanonicalPath(), + unexpandedWildcardClasspath.toString()}; + return jarCp; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java index 0d3df2d0ce0..29ffbe19439 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java @@ -94,7 +94,7 @@ public final class Classpath { final String tmpJarPath; try { tmpJarPath = FileUtil.createJarWithClassPath(classPath, workingDir, - System.getenv()); + System.getenv())[0]; } catch (IOException e) { terminate(1, "I/O error creating jar: " + e.getMessage()); return; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index 2a039887c0f..41794b85314 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -1036,8 +1036,10 @@ public class TestFileUtil { List classPaths = Arrays.asList("", "cp1.jar", "cp2.jar", wildcardPath, "cp3.jar", nonExistentSubdir); String inputClassPath = StringUtils.join(File.pathSeparator, classPaths); - String classPathJar = FileUtil.createJarWithClassPath(inputClassPath, + String[] jarCp = FileUtil.createJarWithClassPath(inputClassPath + File.pathSeparator + "unexpandedwildcard/*", new Path(tmp.getCanonicalPath()), System.getenv()); + String classPathJar = jarCp[0]; + assertNotEquals("Unexpanded wildcard was not placed in extra classpath", jarCp[1].indexOf("unexpanded"), -1); // verify classpath by reading manifest from jar file JarFile jarFile = null; diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 34d275dd6f7..efab925b396 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -678,6 +678,9 @@ Release 2.6.0 - UNRELEASED YARN-2717. Avoided duplicate logging when container logs are not found. (Xuan Gong via zjshen) + YARN-2720. Windows: Wildcard classpath variables not expanded against + resources contained in archives. (Craig Welch via cnauroth) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index fc6c64f5303..578a9a20421 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -148,10 +148,12 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor { // Passing CLASSPATH explicitly is *way* too long for command line. String classPath = System.getProperty("java.class.path"); Map env = new HashMap(System.getenv()); - String classPathJar = FileUtil.createJarWithClassPath(classPath, appStorageDir, env); + String[] jarCp = FileUtil.createJarWithClassPath(classPath, appStorageDir, env); + String classPathJar = jarCp[0]; localizeClasspathJar(new Path(classPathJar), user); + String replacementClassPath = classPathJar + jarCp[1]; command.add("-classpath"); - command.add(classPathJar); + command.add(replacementClassPath); String javaLibPath = System.getProperty("java.library.path"); if (javaLibPath != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index f87ed6a1252..30abef55281 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -764,11 +764,13 @@ public class ContainerLaunch implements Callable { System.getenv()); mergedEnv.putAll(environment); - String classPathJar = FileUtil.createJarWithClassPath( + String[] jarCp = FileUtil.createJarWithClassPath( newClassPath.toString(), pwd, mergedEnv); + String classPathJar = jarCp[0]; // In a secure cluster the classpath jar must be localized to grant access this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser()); - environment.put(Environment.CLASSPATH.name(), classPathJar); + String replacementClassPath = classPathJar + jarCp[1]; + environment.put(Environment.CLASSPATH.name(), replacementClassPath); } } // put AuxiliaryService data to environment