Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13068. Clean up RunJar and related test class. (Contributed by Andras Bokor)

This commit is contained in:
Arpit Agarwal 2016-05-04 09:49:33 -07:00
parent 36972d61d1
commit f343d91ecc
2 changed files with 80 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
View File

@ -23,7 +23,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Array;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
@ -40,7 +39,6 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.io.IOUtils;
import org.slf4j.Logger;
@ -53,7 +51,7 @@ public class RunJar {
private static final Logger LOG = LoggerFactory.getLogger(RunJar.class);
/** Pattern that matches any string */
/** Pattern that matches any string. */
public static final Pattern MATCH_ANY = Pattern.compile(".*");
/**
@ -76,10 +74,21 @@ public class RunJar {
public static final String HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES =
"HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES";
/**
* Buffer size for copy the content of compressed file to new file.
*/
private static final int BUFFER_SIZE = 8_192;
/**
* Unpack a jar file into a directory.
*
* This version unpacks all files inside the jar regardless of filename.
*
* @param jarFile the .jar file to unpack
* @param toDir the destination directory into which to unpack the jar
*
* @throws IOException if an I/O error has occurred or toDir
* cannot be created and does not already exist
*/
public static void unJar(File jarFile, File toDir) throws IOException {
unJar(jarFile, toDir, MATCH_ANY);
@ -92,47 +101,43 @@ public static void unJar(File jarFile, File toDir) throws IOException {
* @param jarFile the .jar file to unpack
* @param toDir the destination directory into which to unpack the jar
* @param unpackRegex the pattern to match jar entries against
*
* @throws IOException if an I/O error has occurred or toDir
* cannot be created and does not already exist
*/
public static void unJar(File jarFile, File toDir, Pattern unpackRegex)
throws IOException {
JarFile jar = new JarFile(jarFile);
try {
throws IOException {
try (JarFile jar = new JarFile(jarFile)) {
int numOfFailedLastModifiedSet = 0;
Enumeration<JarEntry> entries = jar.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
if (!entry.isDirectory() &&
unpackRegex.matcher(entry.getName()).matches()) {
InputStream in = jar.getInputStream(entry);
try {
try (InputStream in = jar.getInputStream(entry)) {
File file = new File(toDir, entry.getName());
ensureDirectory(file.getParentFile());
OutputStream out = new FileOutputStream(file);
try {
IOUtils.copyBytes(in, out, 8192);
} finally {
out.close();
try (OutputStream out = new FileOutputStream(file)) {
IOUtils.copyBytes(in, out, BUFFER_SIZE);
}
if (!file.setLastModified(entry.getTime())) {
numOfFailedLastModifiedSet++;
}
} finally {
in.close();
}
}
}
if (numOfFailedLastModifiedSet > 0) {
LOG.warn("Could not set last modfied time for {} file(s)",
numOfFailedLastModifiedSet);
numOfFailedLastModifiedSet);
}
} finally {
jar.close();
}
}
/**
* Ensure the existence of a given directory.
*
* @param dir Directory to check
*
* @throws IOException if it cannot be created and does not already exist
*/
private static void ensureDirectory(File dir) throws IOException {
@ -169,7 +174,7 @@ public void run(String[] args) throws Throwable {
JarFile jarFile;
try {
jarFile = new JarFile(fileName);
} catch(IOException io) {
} catch (IOException io) {
throw new IOException("Error opening job jar: " + fileName)
.initCause(io);
}
@ -193,11 +198,11 @@ public void run(String[] args) throws Throwable {
ensureDirectory(tmpDir);
final File workDir;
try {
try {
workDir = File.createTempFile("hadoop-unjar", "", tmpDir);
} catch (IOException ioe) {
// If user has insufficient perms to write to tmpDir, default
// "Permission denied" message doesn't specify a filename.
// If user has insufficient perms to write to tmpDir, default
// "Permission denied" message doesn't specify a filename.
System.err.println("Error creating temp dir in java.io.tmpdir "
+ tmpDir + " due to " + ioe.getMessage());
System.exit(-1);
@ -211,12 +216,12 @@ public void run(String[] args) throws Throwable {
ensureDirectory(workDir);
ShutdownHookManager.get().addShutdownHook(
new Runnable() {
@Override
public void run() {
FileUtil.fullyDelete(workDir);
}
}, SHUTDOWN_HOOK_PRIORITY);
new Runnable() {
@Override
public void run() {
FileUtil.fullyDelete(workDir);
}
}, SHUTDOWN_HOOK_PRIORITY);
unJar(file, workDir);
@ -225,13 +230,13 @@ public void run() {
Thread.currentThread().setContextClassLoader(loader);
Class<?> mainClass = Class.forName(mainClassName, true, loader);
Method main = mainClass.getMethod("main", new Class[] {
Array.newInstance(String.class, 0).getClass()
});
String[] newArgs = Arrays.asList(args)
.subList(firstArg, args.length).toArray(new String[0]);
Method main = mainClass.getMethod("main", String[].class);
List<String> newArgsSubList = Arrays.asList(args)
.subList(firstArg, args.length);
String[] newArgs = newArgsSubList
.toArray(new String[newArgsSubList.size()]);
try {
main.invoke(null, new Object[] { newArgs });
main.invoke(null, new Object[] {newArgs});
} catch (InvocationTargetException e) {
throw e.getTargetException();
}
@ -251,10 +256,10 @@ private ClassLoader createClassLoader(File file, final File workDir)
// see if the client classloader is enabled
if (useClientClassLoader()) {
StringBuilder sb = new StringBuilder();
sb.append(workDir+"/").
sb.append(workDir).append("/").
append(File.pathSeparator).append(file).
append(File.pathSeparator).append(workDir+"/classes/").
append(File.pathSeparator).append(workDir+"/lib/*");
append(File.pathSeparator).append(workDir).append("/classes/").
append(File.pathSeparator).append(workDir).append("/lib/*");
// HADOOP_CLASSPATH is added to the client classpath
String hadoopClasspath = getHadoopClasspath();
if (hadoopClasspath != null && !hadoopClasspath.isEmpty()) {
@ -270,18 +275,18 @@ private ClassLoader createClassLoader(File file, final File workDir)
loader = new ApplicationClassLoader(clientClasspath,
getClass().getClassLoader(), systemClassesList);
} else {
List<URL> classPath = new ArrayList<URL>();
classPath.add(new File(workDir+"/").toURI().toURL());
List<URL> classPath = new ArrayList<>();
classPath.add(new File(workDir + "/").toURI().toURL());
classPath.add(file.toURI().toURL());
classPath.add(new File(workDir, "classes/").toURI().toURL());
File[] libs = new File(workDir, "lib").listFiles();
if (libs != null) {
for (int i = 0; i < libs.length; i++) {
classPath.add(libs[i].toURI().toURL());
for (File lib : libs) {
classPath.add(lib.toURI().toURL());
}
}
// create a normal parent-delegating classloader
loader = new URLClassLoader(classPath.toArray(new URL[0]));
loader = new URLClassLoader(classPath.toArray(new URL[classPath.size()]));
}
return loader;
}

62
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
View File

@ -17,6 +17,9 @@
*/
package org.apache.hadoop.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@ -29,15 +32,16 @@
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import junit.framework.TestCase;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class TestRunJar extends TestCase {
public class TestRunJar {
private static final String FOOBAR_TXT = "foobar.txt";
private static final String FOOBAZ_TXT = "foobaz.txt";
private static final int BUFF_SIZE = 2048;
private File TEST_ROOT_DIR;
private static final String TEST_JAR_NAME="test-runjar.jar";
@ -45,9 +49,8 @@ public class TestRunJar extends TestCase {
private static final long MOCKED_NOW = 1_460_389_972_000L;
private static final long MOCKED_NOW_PLUS_TWO_SEC = MOCKED_NOW + 2_000;
@Override
@Before
protected void setUp() throws Exception {
public void setUp() throws Exception {
TEST_ROOT_DIR = GenericTestUtils.getTestDir(getClass().getSimpleName());
if (!TEST_ROOT_DIR.exists()) {
TEST_ROOT_DIR.mkdirs();
@ -56,9 +59,8 @@ protected void setUp() throws Exception {
makeTestJar();
}
@Override
@After
protected void tearDown() {
public void tearDown() {
FileUtil.fullyDelete(TEST_ROOT_DIR);
}
@ -70,11 +72,11 @@ private void makeTestJar() throws IOException {
File jarFile = new File(TEST_ROOT_DIR, TEST_JAR_NAME);
JarOutputStream jstream =
new JarOutputStream(new FileOutputStream(jarFile));
ZipEntry zipEntry1 = new ZipEntry("foobar.txt");
ZipEntry zipEntry1 = new ZipEntry(FOOBAR_TXT);
zipEntry1.setTime(MOCKED_NOW);
jstream.putNextEntry(zipEntry1);
jstream.closeEntry();
ZipEntry zipEntry2 = new ZipEntry("foobaz.txt");
ZipEntry zipEntry2 = new ZipEntry(FOOBAZ_TXT);
zipEntry2.setTime(MOCKED_NOW_PLUS_TWO_SEC);
jstream.putNextEntry(zipEntry2);
jstream.closeEntry();
@ -86,50 +88,52 @@ private void makeTestJar() throws IOException {
*/
@Test
public void testUnJar() throws Exception {
File unjarDir = new File(TEST_ROOT_DIR, "unjar-all");
assertFalse("unjar dir shouldn't exist at test start",
new File(unjarDir, "foobar.txt").exists());
File unjarDir = getUnjarDir("unjar-all");
// Unjar everything
RunJar.unJar(new File(TEST_ROOT_DIR, TEST_JAR_NAME),
unjarDir);
assertTrue("foobar unpacked",
new File(unjarDir, "foobar.txt").exists());
new File(unjarDir, TestRunJar.FOOBAR_TXT).exists());
assertTrue("foobaz unpacked",
new File(unjarDir, "foobaz.txt").exists());
new File(unjarDir, FOOBAZ_TXT).exists());
}
/**
* Test unjarring a specific regex
*/
@Test
public void testUnJarWithPattern() throws Exception {
File unjarDir = new File(TEST_ROOT_DIR, "unjar-pattern");
assertFalse("unjar dir shouldn't exist at test start",
new File(unjarDir, "foobar.txt").exists());
File unjarDir = getUnjarDir("unjar-pattern");
// Unjar only a regex
RunJar.unJar(new File(TEST_ROOT_DIR, TEST_JAR_NAME),
unjarDir,
Pattern.compile(".*baz.*"));
assertFalse("foobar not unpacked",
new File(unjarDir, "foobar.txt").exists());
new File(unjarDir, TestRunJar.FOOBAR_TXT).exists());
assertTrue("foobaz unpacked",
new File(unjarDir, "foobaz.txt").exists());
new File(unjarDir, FOOBAZ_TXT).exists());
}
@Test
public void testUnJarDoesNotLooseLastModify() throws Exception {
File unjarDir = new File(TEST_ROOT_DIR, "unjar-lastmod");
assertFalse("unjar dir shouldn't exist at test start",
new File(unjarDir, "foobar.txt").exists());
File unjarDir = getUnjarDir("unjar-lastmod");
// Unjar everything
RunJar.unJar(new File(TEST_ROOT_DIR, TEST_JAR_NAME),
unjarDir);
assertEquals("Last modify time was lost during unJar", MOCKED_NOW, new File(unjarDir, "foobar.txt").lastModified());
assertEquals("Last modify time was lost during unJar", MOCKED_NOW_PLUS_TWO_SEC, new File(unjarDir, "foobaz.txt").lastModified());
String failureMessage = "Last modify time was lost during unJar";
assertEquals(failureMessage, MOCKED_NOW, new File(unjarDir, TestRunJar.FOOBAR_TXT).lastModified());
assertEquals(failureMessage, MOCKED_NOW_PLUS_TWO_SEC, new File(unjarDir, FOOBAZ_TXT).lastModified());
}
private File getUnjarDir(String dirName) {
File unjarDir = new File(TEST_ROOT_DIR, dirName);
assertFalse("unjar dir shouldn't exist at test start",
new File(unjarDir, TestRunJar.FOOBAR_TXT).exists());
return unjarDir;
}
/**
@ -174,10 +178,10 @@ private File makeClassLoaderTestJar(String... clsNames) throws IOException {
ZipEntry entry = new ZipEntry(name);
jstream.putNextEntry(entry);
BufferedInputStream bufInputStream = new BufferedInputStream(
entryInputStream, 2048);
entryInputStream, BUFF_SIZE);
int count;
byte[] data = new byte[2048];
while ((count = bufInputStream.read(data, 0, 2048)) != -1) {
byte[] data = new byte[BUFF_SIZE];
while ((count = bufInputStream.read(data, 0, BUFF_SIZE)) != -1) {
jstream.write(data, 0, count);
}
jstream.closeEntry();