Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-597. Merging change r1482149 from trunk to branch-2. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1494713 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2013-06-19 18:03:13 +00:00
parent 218178ca11
commit 84e902a0d3
2 changed files with 44 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -657,6 +657,9 @@ Release 2.1.0-beta - UNRELEASED
YARN-839. TestContainerLaunch.testContainerEnvVariables fails on Windows.
(Chuan Liu via cnauroth)
YARN-597. TestFSDownload fails on Windows due to dependencies on
tar/gzip/jar tools. (Ivan Mitic via acmurthy)
YARN-795. Fair scheduler queue metrics should subtract allocated vCores from
available vCores. (ywskycn via tucu)

120
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
View File

@ -25,7 +25,6 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.EnumSet;
import java.util.HashMap;
@ -37,11 +36,16 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import junit.framework.Assert;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@ -50,12 +54,10 @@
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@ -123,32 +125,20 @@ static LocalResource createJar(FileContext files, Path p,
static LocalResource createTarFile(FileContext files, Path p, int len,
Random r, LocalResourceVisibility vis) throws IOException,
URISyntaxException {
byte[] bytes = new byte[len];
r.nextBytes(bytes);
File archiveFile = new File(p.toUri().getPath() + ".tar");
archiveFile.createNewFile();
TarArchiveOutputStream out = new TarArchiveOutputStream(
new FileOutputStream(archiveFile));
TarArchiveEntry entry = new TarArchiveEntry(p.getName());
entry.setSize(bytes.length);
out.putArchiveEntry(entry);
out.write(bytes);
out.closeArchiveEntry();
out.close();
FSDataOutputStream outFile = null;
try {
byte[] bytes = new byte[len];
Path tarPath = new Path(p.toString());
outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
r.nextBytes(bytes);
outFile.write(bytes);
} finally {
if (outFile != null)
outFile.close();
}
StringBuffer tarCommand = new StringBuffer();
URI u = new URI(p.getParent().toString());
tarCommand.append("cd '");
tarCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
tarCommand.append("' ; ");
tarCommand.append("tar -czf " + p.getName() + ".tar " + p.getName());
String[] shellCmd = { "bash", "-c", tarCommand.toString() };
ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
shexec.execute();
int exitcode = shexec.getExitCode();
if (exitcode != 0) {
throw new IOException("Error untarring file " + p
+ ". Tar process exited with exit code " + exitcode);
}
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
+ ".tar")));
@ -163,32 +153,18 @@ static LocalResource createTarFile(FileContext files, Path p, int len,
static LocalResource createJarFile(FileContext files, Path p, int len,
Random r, LocalResourceVisibility vis) throws IOException,
URISyntaxException {
byte[] bytes = new byte[len];
r.nextBytes(bytes);
File archiveFile = new File(p.toUri().getPath() + ".jar");
archiveFile.createNewFile();
JarOutputStream out = new JarOutputStream(
new FileOutputStream(archiveFile));
out.putNextEntry(new JarEntry(p.getName()));
out.write(bytes);
out.closeEntry();
out.close();
FSDataOutputStream outFile = null;
try {
byte[] bytes = new byte[len];
Path tarPath = new Path(p.toString());
outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
r.nextBytes(bytes);
outFile.write(bytes);
} finally {
if (outFile != null)
outFile.close();
}
StringBuffer tarCommand = new StringBuffer();
URI u = new URI(p.getParent().toString());
tarCommand.append("cd '");
tarCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
tarCommand.append("' ; ");
tarCommand.append("jar cf " + p.getName() + ".jar " + p.getName());
String[] shellCmd = { "bash", "-c", tarCommand.toString() };
ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
shexec.execute();
int exitcode = shexec.getExitCode();
if (exitcode != 0) {
throw new IOException("Error untarring file " + p
+ ". Tar process exited with exit code " + exitcode);
}
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
+ ".jar")));
@ -203,39 +179,25 @@ static LocalResource createJarFile(FileContext files, Path p, int len,
static LocalResource createZipFile(FileContext files, Path p, int len,
Random r, LocalResourceVisibility vis) throws IOException,
URISyntaxException {
byte[] bytes = new byte[len];
r.nextBytes(bytes);
File archiveFile = new File(p.toUri().getPath() + ".zip");
archiveFile.createNewFile();
ZipOutputStream out = new ZipOutputStream(
new FileOutputStream(archiveFile));
out.putNextEntry(new ZipEntry(p.getName()));
out.write(bytes);
out.closeEntry();
out.close();
FSDataOutputStream outFile = null;
try {
byte[] bytes = new byte[len];
Path tarPath = new Path(p.toString());
outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
r.nextBytes(bytes);
outFile.write(bytes);
} finally {
if (outFile != null)
outFile.close();
}
StringBuffer zipCommand = new StringBuffer();
URI u = new URI(p.getParent().toString());
zipCommand.append("cd '");
zipCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
zipCommand.append("' ; ");
zipCommand.append("gzip " + p.getName());
String[] shellCmd = { "bash", "-c", zipCommand.toString() };
ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
shexec.execute();
int exitcode = shexec.getExitCode();
if (exitcode != 0) {
throw new IOException("Error untarring file " + p
+ ". Tar process exited with exit code " + exitcode);
}
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
+ ".zip")));
ret.setSize(len);
ret.setType(LocalResourceType.ARCHIVE);
ret.setVisibility(vis);
ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".gz"))
ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".zip"))
.getModificationTime());
return ret;
}