Merge -r 1350007:1350008 from trunk to branch. FIXES: HADOOP-8509

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1350012 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -94,6 +94,8 @@ Release 2.0.1-alpha - UNRELEASED
     HADOOP-8433. Don't set HADOOP_LOG_DIR in hadoop-env.sh.
     (Brahma Reddy Battula via eli)
 
+    HADOOP-8509. JarFinder duplicate entry: META-INF/MANIFEST.MF exception (tucu)
+
   BREAKDOWN OF HDFS-3042 SUBTASKS
 
     HADOOP-8220. ZKFailoverController doesn't handle failure to become active

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/JarFinder.java

@@ -15,15 +15,18 @@ package org.apache.hadoop.util;
 
 import com.google.common.base.Preconditions;
 
+import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.text.MessageFormat;
 import java.util.Enumeration;
+import java.util.jar.JarFile;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
 import java.util.zip.ZipEntry;
@@ -37,10 +40,37 @@ import java.util.zip.ZipOutputStream;
  */
 public class JarFinder {
 
-  private static void zipDir(File dir, String relativePath, ZipOutputStream zos)
+  private static void copyToZipStream(InputStream is, ZipEntry entry,
+                              ZipOutputStream zos) throws IOException {
+    zos.putNextEntry(entry);
+    byte[] arr = new byte[4096];
+    int read = is.read(arr);
+    while (read > -1) {
+      zos.write(arr, 0, read);
+      read = is.read(arr);
+    }
+    is.close();
+    zos.closeEntry();
+  }
+
+  public static void jarDir(File dir, String relativePath, ZipOutputStream zos)
     throws IOException {
     Preconditions.checkNotNull(relativePath, "relativePath");
     Preconditions.checkNotNull(zos, "zos");
+
+    // by JAR spec, if there is a manifest, it must be the first entry in the
+    // ZIP.
+    File manifestFile = new File(dir, JarFile.MANIFEST_NAME);
+    ZipEntry manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
+    if (!manifestFile.exists()) {
+      zos.putNextEntry(manifestEntry);
+      new Manifest().write(new BufferedOutputStream(zos));
+      zos.closeEntry();
+    } else {
+      InputStream is = new FileInputStream(manifestFile);
+      copyToZipStream(is, manifestEntry, zos);
+    }
+    zos.closeEntry();
     zipDir(dir, relativePath, zos, true);
     zos.close();
   }
@@ -62,17 +92,12 @@ public class JarFinder {
           zipDir(file, relativePath + f.getName() + "/", zos, false);
         }
         else {
-          ZipEntry anEntry = new ZipEntry(relativePath + f.getName());
+          String path = relativePath + f.getName();
-          zos.putNextEntry(anEntry);
+          if (!path.equals(JarFile.MANIFEST_NAME)) {
-          InputStream is = new FileInputStream(f);
+            ZipEntry anEntry = new ZipEntry(path);
-          byte[] arr = new byte[4096];
+            InputStream is = new FileInputStream(f);
-          int read = is.read(arr);
+            copyToZipStream(is, anEntry, zos);
-          while (read > -1) {
-            zos.write(arr, 0, read);
-            read = is.read(arr);
           }
-          is.close();
-          zos.closeEntry();
         }
       }
     }
@@ -88,9 +113,8 @@ public class JarFinder {
                                                    jarDir));
       }
     }
-    JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile),
+    JarOutputStream zos = new JarOutputStream(new FileOutputStream(jarFile));
-                                              new Manifest());
+    jarDir(dir, "", zos);
-    zipDir(dir, "", zos);
   }
 
   /**
@@ -142,5 +166,4 @@ public class JarFinder {
     }
     return null;
   }
-
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJarFinder.java

@@ -22,21 +22,105 @@ import org.apache.commons.logging.LogFactory;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.text.MessageFormat;
+import java.util.Properties;
+import java.util.jar.JarInputStream;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 
 public class TestJarFinder {
 
   @Test
-  public void testAppend() throws Exception {
+  public void testJar() throws Exception {
 
     //picking a class that is for sure in a JAR in the classpath
     String jar = JarFinder.getJar(LogFactory.class);
     Assert.assertTrue(new File(jar).exists());
+  }
 
+  private static void delete(File file) throws IOException {
+    if (file.getAbsolutePath().length() < 5) {
+      throw new IllegalArgumentException(
+        MessageFormat.format("Path [{0}] is too short, not deleting",
+                             file.getAbsolutePath()));
+    }
+    if (file.exists()) {
+      if (file.isDirectory()) {
+        File[] children = file.listFiles();
+        if (children != null) {
+          for (File child : children) {
+            delete(child);
+          }
+        }
+      }
+      if (!file.delete()) {
+        throw new RuntimeException(
+          MessageFormat.format("Could not delete path [{0}]",
+                               file.getAbsolutePath()));
+      }
+    }
+  }
+
+  @Test
+  public void testExpandedClasspath() throws Exception {
     //picking a class that is for sure in a directory in the classpath
     //in this case the JAR is created on the fly
-    jar = JarFinder.getJar(TestJarFinder.class);
+    String jar = JarFinder.getJar(TestJarFinder.class);
     Assert.assertTrue(new File(jar).exists());
   }
 
+  @Test
+  public void testExistingManifest() throws Exception {
+    File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
+                        TestJarFinder.class.getName() + "-testExistingManifest");
+    delete(dir);
+    dir.mkdirs();
+
+    File metaInfDir = new File(dir, "META-INF");
+    metaInfDir.mkdirs();
+    File manifestFile = new File(metaInfDir, "MANIFEST.MF");
+    Manifest manifest = new Manifest();
+    OutputStream os = new FileOutputStream(manifestFile);
+    manifest.write(os);
+    os.close();
+
+    File propsFile = new File(dir, "props.properties");
+    Writer writer = new FileWriter(propsFile);
+    new Properties().store(writer, "");
+    writer.close();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    JarOutputStream zos = new JarOutputStream(baos);
+    JarFinder.jarDir(dir, "", zos);
+    JarInputStream jis =
+      new JarInputStream(new ByteArrayInputStream(baos.toByteArray()));
+    Assert.assertNotNull(jis.getManifest());
+    jis.close();
+  }
+
+  @Test
+  public void testNoManifest() throws Exception {
+    File dir = new File(System.getProperty("test.build.dir", "target/test-dir"),
+                        TestJarFinder.class.getName() + "-testNoManifest");
+    delete(dir);
+    dir.mkdirs();
+    File propsFile = new File(dir, "props.properties");
+    Writer writer = new FileWriter(propsFile);
+    new Properties().store(writer, "");
+    writer.close();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    JarOutputStream zos = new JarOutputStream(baos);
+    JarFinder.jarDir(dir, "", zos);
+    JarInputStream jis =
+      new JarInputStream(new ByteArrayInputStream(baos.toByteArray()));
+    Assert.assertNotNull(jis.getManifest());
+    jis.close();
+  }
 }