HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -39,6 +40,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.io.IOUtils;
 
 import com.google.common.base.Preconditions;
@@ -155,7 +159,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
     super(parent);
   }
 
-  private void init(Path path, String pathPrefix,
+  private void init(Path pathPattern, String pathPrefix,
       Configuration conf) throws IOException {
     // Copy the jar to the local filesystem
     String parentDirStr =
@@ -173,7 +177,16 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
       }
     }
 
-    FileSystem fs = path.getFileSystem(conf);
+    FileSystem fs = pathPattern.getFileSystem(conf);
+    Path pathPattern1 = fs.isDirectory(pathPattern) ?
+      new Path(pathPattern, "*.jar") : pathPattern;  // append "*.jar" if a directory is specified
+    FileStatus[] fileStatuses = fs.globStatus(pathPattern1);  // return all files that match the pattern
+    if (fileStatuses == null || fileStatuses.length == 0) {  // if no one matches
+      throw new FileNotFoundException(pathPattern1.toString());
+    } else {
+      boolean validFileEncountered = false;
+      for (Path path : FileUtil.stat2Paths(fileStatuses)) {  // for each file that match the pattern
+        if (fs.isFile(path)) {  // only process files, skip for directories
           File dst = new File(parentDirStr, "." + pathPrefix + "."
             + path.getName() + "." + System.currentTimeMillis() + ".jar");
           fs.copyToLocalFile(path, new Path(dst.toString()));
@@ -183,15 +196,17 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
 
           JarFile jarFile = new JarFile(dst.toString());
           try {
-      Enumeration<JarEntry> entries = jarFile.entries();
+            Enumeration<JarEntry> entries = jarFile.entries();  // get entries inside a jar file
             while (entries.hasMoreElements()) {
               JarEntry entry = entries.nextElement();
               Matcher m = libJarPattern.matcher(entry.getName());
               if (m.matches()) {
                 File file = new File(parentDirStr, "." + pathPrefix + "."
                   + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
+                try (FileOutputStream outStream = new FileOutputStream(file)) {
                   IOUtils.copyBytes(jarFile.getInputStream(entry),
-            new FileOutputStream(file), conf, true);
+                    outStream, conf, true);
+                }
                 file.deleteOnExit();
                 addURL(file.toURI().toURL());
               }
@@ -199,6 +214,14 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
           } finally {
             jarFile.close();
           }
+
+          validFileEncountered = true;  // Set to true when encountering a file
+        }
+      }
+      if (validFileEncountered == false) {  // all items returned by globStatus() are directories
+        throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
+      }
+    }
   }
 
   // This method is used in unit test
@@ -238,7 +261,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
       return cl;
     }
 
-    if (!pathStr.endsWith(".jar")) {
+    if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
       throw new IOException(pathStr + ": not a jar file?");
     }
 

hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -108,4 +109,46 @@ public class TestCoprocessorClassLoader {
     }
     fail("Could not find the expected lib jar file");
   }
+
+  // HBASE-14548
+  @Test
+  public void testDirectoryAndWildcard() throws Exception {
+    String testClassName = "TestClass";
+    String dataTestDir = TEST_UTIL.getDataTestDir().toString();
+    System.out.println(dataTestDir);
+    String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
+    ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
+    ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
+    CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
+
+    CoprocessorClassLoader coprocessorClassLoader = null;
+    Path testPath = null;
+
+    // Directory
+    testPath = new Path(localDirContainingJar);
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1", conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+    // Wildcard - *.jar
+    testPath = new Path(localDirContainingJar, "*.jar");
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2", conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+    // Wildcard - *.j*
+    testPath = new Path(localDirContainingJar, "*.j*");
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3", conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+  }
+
+  /**
+   * Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
+   * @param coprocessorClassLoader the CoprocessorClassLoader to verify
+   * @param className the expected class to be loaded by the coprocessorClassLoader
+   * @throws ClassNotFoundException
+   */
+  private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader, String className)
+    throws ClassNotFoundException{
+    assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
+    assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
+  }
 }