SOLR-1449: solrconfig.xml syntax to add classpath elements from outside of instanceDir

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@824364 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -364,6 +364,10 @@ New Features
 
 85. SOLR-1478: Enable sort by Lucene docid.  (ehatcher)
 
+86. SOLR-1449: Add <lib> elements to solrconfig.xml to specifying additional
+    classpath directories and regular expressions. (hossman via yonik)
+
+
 Optimizations
 ----------------------
  1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the

build.xml

@@ -205,7 +205,7 @@
          <path refid="compile.classpath"/>
          <path refid="compile.classpath.solrj"/>
           <fileset dir="contrib">
-            <include name="**/lib/**.jar"/>
+            <include name="**/lib/**/*.jar"/>
           </fileset>
          <pathelement location="${dest}/client/solrj"/>
       </path>
@@ -374,11 +374,7 @@
         description="Runs the core unit tests."
         depends="test-core, test-contrib" />
 
-  <target name="solr-cell-example" depends="compile">
+  <target name="junit" depends="compileTests,dist-contrib">
-    <ant antfile="contrib/extraction/build.xml" inheritall="false" target="example" />
-  </target>
-  
-  <target name="junit" depends="compileTests,solr-cell-example">
     
     <!-- no description so it doesn't show up in -projecthelp -->  
     <mkdir dir="${junit.output.dir}"/>

contrib/clustering/build.xml

@@ -48,6 +48,10 @@
   <target name="clean">
     <delete failonerror="false" dir="${dest}"/>
 
+    <!-- example doesn't create this anymore, but clean it up
+         if it's still there from an old build
+      -->
+    <delete dir="example/lib" />
   </target>
   <target name="clean-downloads">
     <delete>
@@ -112,17 +116,11 @@
     </solr-javac>
   </target>
 
-  <target name="example" depends="build">
+  <target name="example" depends="build,dist">
-    <mkdir dir="${example.local}/lib"/>
+    <!-- this task use to copy lib's but that's no longer needed because
-    <copy file="${dest}/${fullnamever}.jar" todir="${example.local}/lib"/>
+         ../lib and ../lib/downloads are now included explicitly by
-    <copy todir="${example.local}/lib">
+         example/conf/solrconfig.xml
-      <fileset dir="lib">
+      -->
-        <include name="*.jar"/>
-      </fileset>
-      <fileset dir="lib/downloads">
-        <include name="*.jar"/>
-      </fileset>
-    </copy>
   </target>
 
 

contrib/clustering/example/conf/solrconfig.xml

@@ -26,6 +26,13 @@
      -->
   <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
 
+  <lib dir="../../../dist/" regex="apache-solr-clustering-(\d|\.)+-.*\.jar" />
+  <lib dir="../lib" />
+  <!-- these jars are not inlcuded in the release because of their licenses,
+       they will be downlodded when 'ant example' is run
+    -->
+  <lib dir="../lib/downloads/" />
+  
   <!-- Used to specify an alternate directory to hold all index data
        other than the default ./data under the Solr home.
        If replication is in use, this should match the replication configuration. -->

contrib/dataimporthandler/build.xml

@@ -73,6 +73,9 @@
     </delete>
     <!-- Clean up examples -->
     <delete failonerror="false">
+      <!-- we no longer copy things into this directory, but we still clean it up
+           the files are still there from a previous checkout
+        -->
       <fileset dir="${example}/example-DIH/solr/mail/lib" includes="*.jar" />
     </delete>
   </target>
@@ -197,14 +200,11 @@
     </sequential>
   </target>
 
-  <target name="example" depends="build">
+  <target name="example" depends="build,dist">
-    <!-- Copy the jar into example-DIH/solr/mail/lib -->
+    <!--
-    <copy file="target/apache-${ant.project.name}-extras-${version}.jar" todir="${example}/example-DIH/solr/mail/lib"/>
+        this target use to copy libs, but that is no longer needed.
-    <copy todir="${example}/example-DIH/solr/mail/lib">
+        now we just depend on dist to ensure the extra's jar exists.
-      <fileset dir="lib">
+     -->
-        <include name="**/*.jar"/>
-      </fileset>
-    </copy>
   </target>
   
 </project>

contrib/extraction/build.xml

@@ -114,13 +114,7 @@
   </target>
 
   <target name="example" depends="build">
-    <!-- Copy the jar into example/solr/lib -->
+    <!-- :NOOP: this use to copy libs but now we can refer to them by path -->
-    <copy file="${dest}/${fullnamever}.jar" todir="${example}/solr/lib"/>
-    <copy todir="${example}/solr/lib">
-      <fileset dir="lib">
-        <include name="**/*.jar"/>
-      </fileset>
-    </copy>
   </target>
 
   <target name="javadoc">

example/example-DIH/solr/mail/conf/solrconfig.xml

@@ -26,6 +26,9 @@
      -->
   <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
 
+  <lib dir="../../../../contrib/dataimporthandler/lib/" regex=".*jar$" />
+  <lib dir="../../../../dist/" regex="apache-solr-dataimporthandler-extras.*jar" />
+  
   <indexDefaults>
    <!-- Values here affect all index writers and act as a default unless overridden. -->
     <useCompoundFile>false</useCompoundFile>

example/solr/README.txt

@@ -45,7 +45,9 @@ The Solr Home directory typically contains the following subdirectories...
         This directory is optional.  If it exists, Solr will load any Jars
         found in this directory and use them to resolve any "plugins"
         specified in your solrconfig.xml or schema.xml (ie: Analyzers,
-        Request Handlers, etc...)
+        Request Handlers, etc...).  Alternatively you can use the <lib>
+        syntax in solrconfig.xml to direct Solr to your plugins.  See the
+        example solrconfig.xml file for details.
 
    bin/
         This directory is optional.  It is the default location used for

example/solr/conf/solrconfig.xml

@@ -32,6 +32,36 @@
      -->
   <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
 
+  <!-- lib directives can be used to instruct Solr to load an Jars identified
+       and use them to resolve any "plugins" specified in your solrconfig.xml or
+       schema.xml (ie: Analyzers, Request Handlers, etc...).
+
+       All directories and paths are resolved relative the instanceDir.
+
+       If a "./lib" directory exists in your instanceDir, all files found in it
+       are included as if you had used the following syntax...
+       
+              <lib dir="./lib" />
+    -->
+  <!-- A dir option by itself adds any files found in the directory to the
+       classpath, this is useful for including all jars in a directory.
+    -->
+  <lib dir="../../contrib/extraction/lib" />
+  <!-- When a regex is specified in addition to a directory, only the files in that
+       directory which completely match the regex (anchored on both ends)
+       will be included.
+    -->
+  <lib dir="../../dist/" regex="apache-solr-cell-(\d|\.)+-.*\.jar" />
+  <!-- If a dir option (with or without a regex) is used and nothing is found
+       that matches, it will be ignored
+    -->
+  <lib dir="/total/crap/dir/ignored" /> 
+  <!-- an exact path can be used to specify a specific file.  This will cause
+       a serious error to be logged if it can't be loaded.
+    -->
+  <lib path="../a-jar-that-does-not-exist.jar" /> 
+
+  
   <!-- Used to specify an alternate directory to hold all index data
        other than the default ./data under the Solr home.
        If replication is in use, this should match the replication configuration. -->

src/common/org/apache/solr/common/util/FileUtils.java

@@ -37,7 +37,7 @@ public class FileUtils {
    * of "base")
    * </p>
    */
-  public static File resolvePath(File base, String path) throws IOException {
+  public static File resolvePath(File base, String path) {
     File r = new File(path);
     return r.isAbsolute() ? r : new File(base, path);
   }

src/common/org/apache/solr/common/util/RegexFileFilter.java

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.util.regex.*;
+
+/**
+ * Accepts any file whose name matches the pattern
+ * @version $Id$
+ */
+public final class RegexFileFilter implements FileFilter {
+
+  final Pattern pattern;
+  public RegexFileFilter(String regex) {
+    this(Pattern.compile(regex));
+  }
+  public RegexFileFilter(Pattern regex) {
+    pattern = regex;
+  }
+  public boolean accept(File f) {
+    return pattern.matcher(f.getName()).matches();
+  }
+  public String toString() {
+    return "regex:" + pattern.toString();
+  }
+}

src/java/org/apache/solr/core/SolrConfig.java

@@ -17,6 +17,8 @@
 
 package org.apache.solr.core;
 
+import org.apache.solr.common.util.DOMUtil;
+import org.apache.solr.common.util.RegexFileFilter;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.handler.PingRequestHandler;
 import org.apache.solr.handler.component.SearchComponent;
@@ -49,6 +51,7 @@ import javax.xml.xpath.XPathConstants;
 import java.util.*;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
+import java.io.FileFilter;
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -125,6 +128,7 @@ public class SolrConfig extends Config {
   SolrConfig(SolrResourceLoader loader, String name, InputStream is)
   throws ParserConfigurationException, IOException, SAXException {
     super(loader, name, is, "/config/");
+    initLibs();
     defaultIndexConfig = new SolrIndexConfig(this, null, null);
     mainIndexConfig = new SolrIndexConfig(this, "mainIndex", defaultIndexConfig);
     reopenReaders = getBool("mainIndex/reopenReaders", true);
@@ -425,4 +429,31 @@ public class SolrConfig extends Config {
     List<PluginInfo> result = pluginStore.get(type);
     return result == null || result.isEmpty() ? null: result.get(0);
   }
+  
+  private void initLibs() {
+    
+    NodeList nodes = (NodeList) evaluate("lib", XPathConstants.NODESET);
+    if (nodes==null || nodes.getLength()==0)
+      return;
+    
+    log.info("Adding specified lib dirs to ClassLoader");
+    
+     for (int i=0; i<nodes.getLength(); i++) {
+       Node node = nodes.item(i);
+
+       String baseDir = DOMUtil.getAttr(node, "dir");
+       String path = DOMUtil.getAttr(node, "path");
+       if (null != baseDir) {
+         // :TODO: add support for a simpler 'glob' mutually eclusive of regex
+         String regex = DOMUtil.getAttr(node, "regex");
+         FileFilter filter = (null == regex) ? null : new RegexFileFilter(regex);
+         getResourceLoader().addToClassLoader(baseDir, filter);
+       } else if (null != path) {
+         getResourceLoader().addToClassLoader(path);
+       } else {
+         throw new RuntimeException
+           ("lib: missing mandatory attributes: 'dir' or 'path'");
+       }
+     }
+  }
 }

src/java/org/apache/solr/core/SolrResourceLoader.java

@@ -19,6 +19,7 @@ package org.apache.solr.core;
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -42,6 +43,7 @@ import javax.naming.NoInitialContextException;
 import org.apache.solr.analysis.CharFilterFactory;
 import org.apache.solr.analysis.TokenFilterFactory;
 import org.apache.solr.analysis.TokenizerFactory;
+import org.apache.solr.common.util.FileUtils;
 import org.apache.solr.common.ResourceLoader;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.handler.component.SearchComponent;
@@ -63,7 +65,7 @@ public class SolrResourceLoader implements ResourceLoader
   static final String base = "org.apache" + "." + project;
   static final String[] packages = {"","analysis.","schema.","handler.","search.","update.","core.","request.","update.processor.","util.", "spelling.", "handler.component.", "handler.dataimport"};
 
-  private final ClassLoader classLoader;
+  private URLClassLoader classLoader;
   private final String instanceDir;
   private String dataDir;
   
@@ -90,7 +92,10 @@ public class SolrResourceLoader implements ResourceLoader
       this.instanceDir = normalizeDir(instanceDir);
     }
     log.info("Solr home set to '" + this.instanceDir + "'");
-    this.classLoader = createClassLoader(new File(this.instanceDir + "lib/"), parent);
+    
+    this.classLoader = createClassLoader(null, parent);
+    addToClassLoader("./lib/", null);
+    
     this.coreProperties = coreProperties;
   }
 
@@ -107,25 +112,82 @@ public class SolrResourceLoader implements ResourceLoader
     this(instanceDir, parent, null);
   }
 
-  static ClassLoader createClassLoader(File f, ClassLoader loader) {
+  /**
-    if( loader == null ) {
+   * Adds every file/dir found in the baseDir which passes the specified Filter
-      loader = Thread.currentThread().getContextClassLoader();
+   * to the ClassLoader used by this ResourceLoader.  This method <b>MUST</b>
+   * only be called prior to using this ResourceLoader to get any resources, otherwise
+   * it's behavior will be non-deterministic.
+   *
+   * @param baseDir base directory whose children (either jars or directories of
+   *                classes) will be in the classpath, will be resolved relative
+   *                the instance dir.
+   * @param filter The filter files must satisfy, if null all files will be accepted.
+   */
+  void addToClassLoader(final String baseDir, final FileFilter filter) {
+    File base = FileUtils.resolvePath(new File(getInstanceDir()), baseDir);
+    this.classLoader = replaceClassLoader(classLoader, base, filter);
   }
-    if (f.canRead() && f.isDirectory()) {
+  
-      File[] jarFiles = f.listFiles();
+  /**
-      URL[] jars = new URL[jarFiles.length];
+   * Adds the specific file/dir specified to the ClassLoader used by this
+   * ResourceLoader.  This method <b>MUST</b>
+   * only be called prior to using this ResourceLoader to get any resources, otherwise
+   * it's behavior will be non-deterministic.
+   *
+   * @param path A jar file (or directory of classes) to be added to the classpath,
+   *             will be resolved relative the instance dir.
+   */
+  void addToClassLoader(final String path) {
+    final File file = FileUtils.resolvePath(new File(getInstanceDir()), path);
+    if (file.canRead()) {
+      this.classLoader = replaceClassLoader(classLoader, file.getParentFile(),
+                                            new FileFilter() {
+                                              public boolean accept(File pathname) {
+                                                return pathname.equals(file);
+                                              }
+                                            });
+    } else {
+      log.error("Can't find (or read) file to add to classloader: " + file);
+    }
+  }
+  
+  private static URLClassLoader replaceClassLoader(final URLClassLoader oldLoader,
+                                                   final File base,
+                                                   final FileFilter filter) {
+    if (null != base && base.canRead() && base.isDirectory()) {
+      File[] files = base.listFiles(filter);
+      
+      if (null == files || 0 == files.length) return oldLoader;
+      
+      URL[] oldElements = oldLoader.getURLs();
+      URL[] elements = new URL[oldElements.length + files.length];
+      System.arraycopy(oldElements, 0, elements, 0, oldElements.length);
+      
+      for (int j = 0; j < files.length; j++) {
         try {
-        for (int j = 0; j < jarFiles.length; j++) {
+          URL element = files[j].toURI().normalize().toURL();
-          jars[j] = jarFiles[j].toURI().toURL();
+          log.info("Adding '" + element.toString() + "' to classloader");
-          log.info("Adding '" + jars[j].toString() + "' to Solr classloader");
+          elements[oldElements.length + j] = element;
-        }
-        return URLClassLoader.newInstance(jars, loader);
         } catch (MalformedURLException e) {
-        SolrException.log(log,"Can't construct solr lib class loader", e);
+          SolrException.log(log, "Can't add element to classloader: " + files[j], e);
         }
       }
-    log.info("Reusing parent classloader");
+      return URLClassLoader.newInstance(elements, oldLoader.getParent());
-    return loader;
+    }
+    // are we still here?
+    return oldLoader;
+  }
+  
+  /**
+   * Convenience method for getting a new ClassLoader using all files found
+   * in the specified lib directory.
+   */
+  static URLClassLoader createClassLoader(final File libDir, ClassLoader parent) {
+    if ( null == parent ) {
+      parent = Thread.currentThread().getContextClassLoader();
+    }
+    return replaceClassLoader(URLClassLoader.newInstance(new URL[0], parent),
+                              libDir, null);
   }
   
   public SolrResourceLoader( String instanceDir )
@@ -553,6 +615,4 @@ public class SolrResourceLoader implements ResourceLoader
     }
     throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, builder.toString() );
   }
-
-
 }

src/test/org/apache/solr/core/TestConfig.java

@@ -30,6 +30,7 @@ import org.w3c.dom.NodeList;
 
 import javax.xml.xpath.XPathConstants;
 import java.io.IOException;
+import java.io.InputStream;
 
 public class TestConfig extends AbstractSolrTestCase {
 
@@ -42,6 +43,31 @@ public class TestConfig extends AbstractSolrTestCase {
     return "solrconfig-termindex.xml";
   }
 
+  public void testLib() throws IOException {
+    SolrResourceLoader loader = h.getCore().getResourceLoader();
+    InputStream data = null;
+    String[] expectedFiles = new String[] { "empty-file-main-lib.txt",
+            "empty-file-a1.txt",
+            "empty-file-a2.txt",
+            "empty-file-b1.txt",
+            "empty-file-b2.txt",
+            "empty-file-c1.txt" };
+    for (String f : expectedFiles) {
+      data = loader.openResource(f);
+      assertNotNull("Should have found file " + f, data);
+      data.close();
+    }
+    String[] unexpectedFiles = new String[] { "empty-file-c2.txt",
+            "empty-file-d2.txt" };
+    for (String f : unexpectedFiles) {
+      data = null;
+      try {
+        data = loader.openResource(f);
+      } catch (Exception e) { /* :NOOP: (un)expected */ }
+      assertNull("should not have been able to find " + f, data);
+    }
+  }
+
   public void testJavaProperty() {
     // property values defined in build.xml
 

src/test/test-files/lib-dirs/README

@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Items under this directory are used by TestConfig.testLibs()

src/test/test-files/lib-dirs/a/a1/empty-file-a1.txt

@@ -0,0 +1 @@
+

src/test/test-files/lib-dirs/a/a2/empty-file-a2.txt

@@ -0,0 +1 @@
+

src/test/test-files/lib-dirs/b/b1/empty-file-b1.txt

@@ -0,0 +1 @@
+

src/test/test-files/lib-dirs/b/b2/empty-file-b2.txt

@@ -0,0 +1 @@
+

src/test/test-files/lib-dirs/c/c1/empty-file-c1.txt

@@ -0,0 +1 @@
+

src/test/test-files/lib-dirs/c/c2/empty-file-c2.txt

@@ -0,0 +1 @@
+

src/test/test-files/lib-dirs/d/d1/empty-file-d1.txt

@@ -0,0 +1 @@
+

src/test/test-files/lib-dirs/d/d2/empty-file-d2.txt

@@ -0,0 +1 @@
+

src/test/test-files/solr/conf/solrconfig-termindex.xml

@@ -24,6 +24,12 @@
 
 <config>
 
+  <!-- see TestConfig.testLib() -->
+  <lib dir="../lib-dirs/a" />
+  <lib dir="../lib-dirs/b" regex="b." />
+  <lib dir="../lib-dirs/c" regex="c1" />
+  <lib path="../lib-dirs/d/d1/" />
+  
   <jmx />
 
   <!-- Used to specify an alternate directory to hold all index data.

src/test/test-files/solr/lib/README

@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+Items under this directory are used by TestConfig.testLibs()

src/test/test-files/solr/lib/classes/empty-file-main-lib.txt

@@ -0,0 +1 @@
+