HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -38,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.io.IOUtils;
 
@@ -142,7 +145,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
     super(parent);
   }
 
-  private void init(Path path, String pathPrefix,
+  private void init(Path pathPattern, String pathPrefix,
       Configuration conf) throws IOException {
     // Copy the jar to the local filesystem
     String parentDirStr =
@@ -160,33 +163,50 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
       }
     }
 
-    FileSystem fs = path.getFileSystem(conf);
+    FileSystem fs = pathPattern.getFileSystem(conf);
-    File dst = new File(parentDirStr, "." + pathPrefix + "."
+    Path pathPattern1 = fs.isDirectory(pathPattern) ?
-      + path.getName() + "." + System.currentTimeMillis() + ".jar");
+      new Path(pathPattern, "*.jar") : pathPattern;  // append "*.jar" if a directory is specified
-    fs.copyToLocalFile(path, new Path(dst.toString()));
+    FileStatus[] fileStatuses = fs.globStatus(pathPattern1);  // return all files that match the pattern
-    dst.deleteOnExit();
+    if (fileStatuses == null || fileStatuses.length == 0) {  // if no one matches
+      throw new FileNotFoundException(pathPattern1.toString());
+    } else {
+      boolean validFileEncountered = false;
+      for (Path path : FileUtil.stat2Paths(fileStatuses)) {  // for each file that match the pattern
+        if (fs.isFile(path)) {  // only process files, skip for directories
+          File dst = new File(parentDirStr, "." + pathPrefix + "."
+            + path.getName() + "." + System.currentTimeMillis() + ".jar");
+          fs.copyToLocalFile(path, new Path(dst.toString()));
+          dst.deleteOnExit();
 
-    addURL(dst.getCanonicalFile().toURI().toURL());
+          addURL(dst.getCanonicalFile().toURI().toURL());
 
-    JarFile jarFile = new JarFile(dst.toString());
+          JarFile jarFile = new JarFile(dst.toString());
-    try {
+          try {
-      Enumeration<JarEntry> entries = jarFile.entries();
+            Enumeration<JarEntry> entries = jarFile.entries();  // get entries inside a jar file
-      while (entries.hasMoreElements()) {
+            while (entries.hasMoreElements()) {
-        JarEntry entry = entries.nextElement();
+              JarEntry entry = entries.nextElement();
-        Matcher m = libJarPattern.matcher(entry.getName());
+              Matcher m = libJarPattern.matcher(entry.getName());
-        if (m.matches()) {
+              if (m.matches()) {
-          File file = new File(parentDirStr, "." + pathPrefix + "."
+                File file = new File(parentDirStr, "." + pathPrefix + "."
-            + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
+                  + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
-          try (FileOutputStream outStream = new FileOutputStream(file)) {
+                try (FileOutputStream outStream = new FileOutputStream(file)) {
-            IOUtils.copyBytes(jarFile.getInputStream(entry),
+                  IOUtils.copyBytes(jarFile.getInputStream(entry),
-              outStream, conf, true);
+                    outStream, conf, true);
+                }
+                file.deleteOnExit();
+                addURL(file.toURI().toURL());
+              }
+            }
+          } finally {
+            jarFile.close();
           }
-          file.deleteOnExit();
+
-          addURL(file.toURI().toURL());
+          validFileEncountered = true;  // Set to true when encountering a file
         }
       }
-    } finally {
+      if (validFileEncountered == false) {  // all items returned by globStatus() are directories
-      jarFile.close();
+        throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
+      }
     }
   }
 
@@ -227,7 +247,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
       return cl;
     }
 
-    if (!pathStr.endsWith(".jar")) {
+    if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
       throw new IOException(pathStr + ": not a jar file?");
     }
 

hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -109,4 +110,46 @@ public class TestCoprocessorClassLoader {
     }
     fail("Could not find the expected lib jar file");
   }
+
+  // HBASE-14548
+  @Test
+  public void testDirectoryAndWildcard() throws Exception {
+    String testClassName = "TestClass";
+    String dataTestDir = TEST_UTIL.getDataTestDir().toString();
+    System.out.println(dataTestDir);
+    String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
+    ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
+    ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
+    CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
+
+    CoprocessorClassLoader coprocessorClassLoader = null;
+    Path testPath = null;
+
+    // Directory
+    testPath = new Path(localDirContainingJar);
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1", conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+    // Wildcard - *.jar
+    testPath = new Path(localDirContainingJar, "*.jar");
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2", conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+    // Wildcard - *.j*
+    testPath = new Path(localDirContainingJar, "*.j*");
+    coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3", conf);
+    verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+  }
+
+  /**
+   * Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
+   * @param coprocessorClassLoader the CoprocessorClassLoader to verify
+   * @param className the expected class to be loaded by the coprocessorClassLoader
+   * @throws ClassNotFoundException
+   */
+  private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader, String className)
+    throws ClassNotFoundException{
+    assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
+    assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
+  }
 }

src/main/asciidoc/_chapters/cp.adoc

@@ -320,7 +320,14 @@ The value contains four pieces of information which are separated by the pipe (`
 * File path: The jar file containing the Coprocessor implementation must be in a location where
 all region servers can read it. +
 You could copy the file onto the local disk on each region server, but it is recommended to store
-it in HDFS.
+it in HDFS. +
+https://issues.apache.org/jira/browse/HBASE-14548[HBASE-14548] allows a directory containing the jars
+or some wildcards to be specified, such as: hdfs://<namenode>:<port>/user/<hadoop-user>/ or
+hdfs://<namenode>:<port>/user/<hadoop-user>/*.jar. Please note that if a directory is specified,
+all jar files(.jar) directly in the directory are added,
+but it does not search files in the subtree rooted in the directory.
+And do not contain any wildcard if you would like to specify a directory.
+This enhancement applies to the ways of using the JAVA API as well.
 * Class name: The full class name of the Coprocessor.
 * Priority: An integer. The framework will determine the execution sequence of all configured
 observers registered at the same hook using priorities. This field can be left blank. In that