YARN-597. Merging change r1482149 from trunk to branch-2.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1494713 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -657,6 +657,9 @@ Release 2.1.0-beta - UNRELEASED
     YARN-839. TestContainerLaunch.testContainerEnvVariables fails on Windows.
     (Chuan Liu via cnauroth)
 
+    YARN-597. TestFSDownload fails on Windows due to dependencies on
+    tar/gzip/jar tools. (Ivan Mitic via acmurthy) 
+
     YARN-795. Fair scheduler queue metrics should subtract allocated vCores from 
     available vCores. (ywskycn via tucu) 
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java

@@ -25,7 +25,6 @@ import static org.junit.Assert.assertEquals;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -37,11 +36,16 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 import junit.framework.Assert;
 
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -50,12 +54,10 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -123,32 +125,20 @@ public class TestFSDownload {
   static LocalResource createTarFile(FileContext files, Path p, int len,
       Random r, LocalResourceVisibility vis) throws IOException,
       URISyntaxException {
+    byte[] bytes = new byte[len];
+    r.nextBytes(bytes);
+
+    File archiveFile = new File(p.toUri().getPath() + ".tar");
+    archiveFile.createNewFile();
+    TarArchiveOutputStream out = new TarArchiveOutputStream(
+        new FileOutputStream(archiveFile));
+    TarArchiveEntry entry = new TarArchiveEntry(p.getName());
+    entry.setSize(bytes.length);
+    out.putArchiveEntry(entry);
+    out.write(bytes);
+    out.closeArchiveEntry();
+    out.close();
 
-    FSDataOutputStream outFile = null;
-    try {
-      byte[] bytes = new byte[len];
-      Path tarPath = new Path(p.toString());
-      outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
-      r.nextBytes(bytes);
-      outFile.write(bytes);
-    } finally {
-      if (outFile != null)
-        outFile.close();
-    }
-    StringBuffer tarCommand = new StringBuffer();
-    URI u = new URI(p.getParent().toString());
-    tarCommand.append("cd '");
-    tarCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
-    tarCommand.append("' ; ");
-    tarCommand.append("tar -czf " + p.getName() + ".tar " + p.getName());
-    String[] shellCmd = { "bash", "-c", tarCommand.toString() };
-    ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
-    shexec.execute();
-    int exitcode = shexec.getExitCode();
-    if (exitcode != 0) {
-      throw new IOException("Error untarring file " + p
-          + ". Tar process exited with exit code " + exitcode);
-    }
     LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
     ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
         + ".tar")));
@@ -163,32 +153,18 @@ public class TestFSDownload {
   static LocalResource createJarFile(FileContext files, Path p, int len,
       Random r, LocalResourceVisibility vis) throws IOException,
       URISyntaxException {
+    byte[] bytes = new byte[len];
+    r.nextBytes(bytes);
+
+    File archiveFile = new File(p.toUri().getPath() + ".jar");
+    archiveFile.createNewFile();
+    JarOutputStream out = new JarOutputStream(
+        new FileOutputStream(archiveFile));
+    out.putNextEntry(new JarEntry(p.getName()));
+    out.write(bytes);
+    out.closeEntry();
+    out.close();
 
-    FSDataOutputStream outFile = null;
-    try {
-      byte[] bytes = new byte[len];
-      Path tarPath = new Path(p.toString());
-      outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
-      r.nextBytes(bytes);
-      outFile.write(bytes);
-    } finally {
-      if (outFile != null)
-        outFile.close();
-    }
-    StringBuffer tarCommand = new StringBuffer();
-    URI u = new URI(p.getParent().toString());
-    tarCommand.append("cd '");
-    tarCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
-    tarCommand.append("' ; ");
-    tarCommand.append("jar cf " + p.getName() + ".jar " + p.getName());
-    String[] shellCmd = { "bash", "-c", tarCommand.toString() };
-    ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
-    shexec.execute();
-    int exitcode = shexec.getExitCode();
-    if (exitcode != 0) {
-      throw new IOException("Error untarring file " + p
-          + ". Tar process exited with exit code " + exitcode);
-    }
     LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
     ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
         + ".jar")));
@@ -203,39 +179,25 @@ public class TestFSDownload {
   static LocalResource createZipFile(FileContext files, Path p, int len,
       Random r, LocalResourceVisibility vis) throws IOException,
       URISyntaxException {
+    byte[] bytes = new byte[len];
+    r.nextBytes(bytes);
+
+    File archiveFile = new File(p.toUri().getPath() + ".zip");
+    archiveFile.createNewFile();
+    ZipOutputStream out = new ZipOutputStream(
+        new FileOutputStream(archiveFile));
+    out.putNextEntry(new ZipEntry(p.getName()));
+    out.write(bytes);
+    out.closeEntry();
+    out.close();
 
-    FSDataOutputStream outFile = null;
-    try {
-      byte[] bytes = new byte[len];
-      Path tarPath = new Path(p.toString());
-      outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
-      r.nextBytes(bytes);
-      outFile.write(bytes);
-    } finally {
-      if (outFile != null)
-        outFile.close();
-    }
-    StringBuffer zipCommand = new StringBuffer();
-    URI u = new URI(p.getParent().toString());
-    zipCommand.append("cd '");
-    zipCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
-    zipCommand.append("' ; ");
-    zipCommand.append("gzip " + p.getName());
-    String[] shellCmd = { "bash", "-c", zipCommand.toString() };
-    ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
-    shexec.execute();
-    int exitcode = shexec.getExitCode();
-    if (exitcode != 0) {
-      throw new IOException("Error untarring file " + p
-          + ". Tar process exited with exit code " + exitcode);
-    }
     LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
     ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
         + ".zip")));
     ret.setSize(len);
     ret.setType(LocalResourceType.ARCHIVE);
     ret.setVisibility(vis);
-    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".gz"))
+    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".zip"))
         .getModificationTime());
     return ret;
   }