SOLR-1449: solrconfig.xml syntax to add classpath elements from outside of instanceDir

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@824364 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2009-10-12 14:41:51 +00:00
parent fa623ae3c2
commit afcfa3c1c2
26 changed files with 426 additions and 181 deletions

View File

@ -364,6 +364,10 @@ New Features
85. SOLR-1478: Enable sort by Lucene docid. (ehatcher)
86. SOLR-1449: Add <lib> elements to solrconfig.xml to specifying additional
classpath directories and regular expressions. (hossman via yonik)
Optimizations
----------------------
1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the

View File

@ -205,7 +205,7 @@
<path refid="compile.classpath"/>
<path refid="compile.classpath.solrj"/>
<fileset dir="contrib">
<include name="**/lib/**.jar"/>
<include name="**/lib/**/*.jar"/>
</fileset>
<pathelement location="${dest}/client/solrj"/>
</path>
@ -374,11 +374,7 @@
description="Runs the core unit tests."
depends="test-core, test-contrib" />
<target name="solr-cell-example" depends="compile">
<ant antfile="contrib/extraction/build.xml" inheritall="false" target="example" />
</target>
<target name="junit" depends="compileTests,solr-cell-example">
<target name="junit" depends="compileTests,dist-contrib">
<!-- no description so it doesn't show up in -projecthelp -->
<mkdir dir="${junit.output.dir}"/>

View File

@ -48,6 +48,10 @@
<target name="clean">
<delete failonerror="false" dir="${dest}"/>
<!-- example doesn't create this anymore, but clean it up
if it's still there from an old build
-->
<delete dir="example/lib" />
</target>
<target name="clean-downloads">
<delete>
@ -112,17 +116,11 @@
</solr-javac>
</target>
<target name="example" depends="build">
<mkdir dir="${example.local}/lib"/>
<copy file="${dest}/${fullnamever}.jar" todir="${example.local}/lib"/>
<copy todir="${example.local}/lib">
<fileset dir="lib">
<include name="*.jar"/>
</fileset>
<fileset dir="lib/downloads">
<include name="*.jar"/>
</fileset>
</copy>
<target name="example" depends="build,dist">
<!-- this task use to copy lib's but that's no longer needed because
../lib and ../lib/downloads are now included explicitly by
example/conf/solrconfig.xml
-->
</target>

View File

@ -26,6 +26,13 @@
-->
<abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<lib dir="../../../dist/" regex="apache-solr-clustering-(\d|\.)+-.*\.jar" />
<lib dir="../lib" />
<!-- these jars are not inlcuded in the release because of their licenses,
they will be downlodded when 'ant example' is run
-->
<lib dir="../lib/downloads/" />
<!-- Used to specify an alternate directory to hold all index data
other than the default ./data under the Solr home.
If replication is in use, this should match the replication configuration. -->

View File

@ -73,6 +73,9 @@
</delete>
<!-- Clean up examples -->
<delete failonerror="false">
<!-- we no longer copy things into this directory, but we still clean it up
the files are still there from a previous checkout
-->
<fileset dir="${example}/example-DIH/solr/mail/lib" includes="*.jar" />
</delete>
</target>
@ -197,14 +200,11 @@
</sequential>
</target>
<target name="example" depends="build">
<!-- Copy the jar into example-DIH/solr/mail/lib -->
<copy file="target/apache-${ant.project.name}-extras-${version}.jar" todir="${example}/example-DIH/solr/mail/lib"/>
<copy todir="${example}/example-DIH/solr/mail/lib">
<fileset dir="lib">
<include name="**/*.jar"/>
</fileset>
</copy>
<target name="example" depends="build,dist">
<!--
this target use to copy libs, but that is no longer needed.
now we just depend on dist to ensure the extra's jar exists.
-->
</target>
</project>

View File

@ -114,13 +114,7 @@
</target>
<target name="example" depends="build">
<!-- Copy the jar into example/solr/lib -->
<copy file="${dest}/${fullnamever}.jar" todir="${example}/solr/lib"/>
<copy todir="${example}/solr/lib">
<fileset dir="lib">
<include name="**/*.jar"/>
</fileset>
</copy>
<!-- :NOOP: this use to copy libs but now we can refer to them by path -->
</target>
<target name="javadoc">

View File

@ -26,6 +26,9 @@
-->
<abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<lib dir="../../../../contrib/dataimporthandler/lib/" regex=".*jar$" />
<lib dir="../../../../dist/" regex="apache-solr-dataimporthandler-extras.*jar" />
<indexDefaults>
<!-- Values here affect all index writers and act as a default unless overridden. -->
<useCompoundFile>false</useCompoundFile>

View File

@ -45,7 +45,9 @@ The Solr Home directory typically contains the following subdirectories...
This directory is optional. If it exists, Solr will load any Jars
found in this directory and use them to resolve any "plugins"
specified in your solrconfig.xml or schema.xml (ie: Analyzers,
Request Handlers, etc...)
Request Handlers, etc...). Alternatively you can use the <lib>
syntax in solrconfig.xml to direct Solr to your plugins. See the
example solrconfig.xml file for details.
bin/
This directory is optional. It is the default location used for

View File

@ -32,6 +32,36 @@
-->
<abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
<!-- lib directives can be used to instruct Solr to load an Jars identified
and use them to resolve any "plugins" specified in your solrconfig.xml or
schema.xml (ie: Analyzers, Request Handlers, etc...).
All directories and paths are resolved relative the instanceDir.
If a "./lib" directory exists in your instanceDir, all files found in it
are included as if you had used the following syntax...
<lib dir="./lib" />
-->
<!-- A dir option by itself adds any files found in the directory to the
classpath, this is useful for including all jars in a directory.
-->
<lib dir="../../contrib/extraction/lib" />
<!-- When a regex is specified in addition to a directory, only the files in that
directory which completely match the regex (anchored on both ends)
will be included.
-->
<lib dir="../../dist/" regex="apache-solr-cell-(\d|\.)+-.*\.jar" />
<!-- If a dir option (with or without a regex) is used and nothing is found
that matches, it will be ignored
-->
<lib dir="/total/crap/dir/ignored" />
<!-- an exact path can be used to specify a specific file. This will cause
a serious error to be logged if it can't be loaded.
-->
<lib path="../a-jar-that-does-not-exist.jar" />
<!-- Used to specify an alternate directory to hold all index data
other than the default ./data under the Solr home.
If replication is in use, this should match the replication configuration. -->

View File

@ -37,7 +37,7 @@ public class FileUtils {
* of "base")
* </p>
*/
public static File resolvePath(File base, String path) throws IOException {
public static File resolvePath(File base, String path) {
File r = new File(path);
return r.isAbsolute() ? r : new File(base, path);
}

View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.util;
import java.io.File;
import java.io.FileFilter;
import java.util.regex.*;
/**
* Accepts any file whose name matches the pattern
* @version $Id$
*/
public final class RegexFileFilter implements FileFilter {
final Pattern pattern;
public RegexFileFilter(String regex) {
this(Pattern.compile(regex));
}
public RegexFileFilter(Pattern regex) {
pattern = regex;
}
public boolean accept(File f) {
return pattern.matcher(f.getName()).matches();
}
public String toString() {
return "regex:" + pattern.toString();
}
}

View File

@ -17,6 +17,8 @@
package org.apache.solr.core;
import org.apache.solr.common.util.DOMUtil;
import org.apache.solr.common.util.RegexFileFilter;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.PingRequestHandler;
import org.apache.solr.handler.component.SearchComponent;
@ -49,6 +51,7 @@ import javax.xml.xpath.XPathConstants;
import java.util.*;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
@ -125,6 +128,7 @@ public class SolrConfig extends Config {
SolrConfig(SolrResourceLoader loader, String name, InputStream is)
throws ParserConfigurationException, IOException, SAXException {
super(loader, name, is, "/config/");
initLibs();
defaultIndexConfig = new SolrIndexConfig(this, null, null);
mainIndexConfig = new SolrIndexConfig(this, "mainIndex", defaultIndexConfig);
reopenReaders = getBool("mainIndex/reopenReaders", true);
@ -425,4 +429,31 @@ public class SolrConfig extends Config {
List<PluginInfo> result = pluginStore.get(type);
return result == null || result.isEmpty() ? null: result.get(0);
}
private void initLibs() {
NodeList nodes = (NodeList) evaluate("lib", XPathConstants.NODESET);
if (nodes==null || nodes.getLength()==0)
return;
log.info("Adding specified lib dirs to ClassLoader");
for (int i=0; i<nodes.getLength(); i++) {
Node node = nodes.item(i);
String baseDir = DOMUtil.getAttr(node, "dir");
String path = DOMUtil.getAttr(node, "path");
if (null != baseDir) {
// :TODO: add support for a simpler 'glob' mutually eclusive of regex
String regex = DOMUtil.getAttr(node, "regex");
FileFilter filter = (null == regex) ? null : new RegexFileFilter(regex);
getResourceLoader().addToClassLoader(baseDir, filter);
} else if (null != path) {
getResourceLoader().addToClassLoader(path);
} else {
throw new RuntimeException
("lib: missing mandatory attributes: 'dir' or 'path'");
}
}
}
}

View File

@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@ -42,6 +43,7 @@ import javax.naming.NoInitialContextException;
import org.apache.solr.analysis.CharFilterFactory;
import org.apache.solr.analysis.TokenFilterFactory;
import org.apache.solr.analysis.TokenizerFactory;
import org.apache.solr.common.util.FileUtils;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.handler.component.SearchComponent;
@ -63,7 +65,7 @@ public class SolrResourceLoader implements ResourceLoader
static final String base = "org.apache" + "." + project;
static final String[] packages = {"","analysis.","schema.","handler.","search.","update.","core.","request.","update.processor.","util.", "spelling.", "handler.component.", "handler.dataimport"};
private final ClassLoader classLoader;
private URLClassLoader classLoader;
private final String instanceDir;
private String dataDir;
@ -90,7 +92,10 @@ public class SolrResourceLoader implements ResourceLoader
this.instanceDir = normalizeDir(instanceDir);
}
log.info("Solr home set to '" + this.instanceDir + "'");
this.classLoader = createClassLoader(new File(this.instanceDir + "lib/"), parent);
this.classLoader = createClassLoader(null, parent);
addToClassLoader("./lib/", null);
this.coreProperties = coreProperties;
}
@ -107,25 +112,82 @@ public class SolrResourceLoader implements ResourceLoader
this(instanceDir, parent, null);
}
static ClassLoader createClassLoader(File f, ClassLoader loader) {
if( loader == null ) {
loader = Thread.currentThread().getContextClassLoader();
/**
* Adds every file/dir found in the baseDir which passes the specified Filter
* to the ClassLoader used by this ResourceLoader. This method <b>MUST</b>
* only be called prior to using this ResourceLoader to get any resources, otherwise
* it's behavior will be non-deterministic.
*
* @param baseDir base directory whose children (either jars or directories of
* classes) will be in the classpath, will be resolved relative
* the instance dir.
* @param filter The filter files must satisfy, if null all files will be accepted.
*/
void addToClassLoader(final String baseDir, final FileFilter filter) {
File base = FileUtils.resolvePath(new File(getInstanceDir()), baseDir);
this.classLoader = replaceClassLoader(classLoader, base, filter);
}
if (f.canRead() && f.isDirectory()) {
File[] jarFiles = f.listFiles();
URL[] jars = new URL[jarFiles.length];
/**
* Adds the specific file/dir specified to the ClassLoader used by this
* ResourceLoader. This method <b>MUST</b>
* only be called prior to using this ResourceLoader to get any resources, otherwise
* it's behavior will be non-deterministic.
*
* @param path A jar file (or directory of classes) to be added to the classpath,
* will be resolved relative the instance dir.
*/
void addToClassLoader(final String path) {
final File file = FileUtils.resolvePath(new File(getInstanceDir()), path);
if (file.canRead()) {
this.classLoader = replaceClassLoader(classLoader, file.getParentFile(),
new FileFilter() {
public boolean accept(File pathname) {
return pathname.equals(file);
}
});
} else {
log.error("Can't find (or read) file to add to classloader: " + file);
}
}
private static URLClassLoader replaceClassLoader(final URLClassLoader oldLoader,
final File base,
final FileFilter filter) {
if (null != base && base.canRead() && base.isDirectory()) {
File[] files = base.listFiles(filter);
if (null == files || 0 == files.length) return oldLoader;
URL[] oldElements = oldLoader.getURLs();
URL[] elements = new URL[oldElements.length + files.length];
System.arraycopy(oldElements, 0, elements, 0, oldElements.length);
for (int j = 0; j < files.length; j++) {
try {
for (int j = 0; j < jarFiles.length; j++) {
jars[j] = jarFiles[j].toURI().toURL();
log.info("Adding '" + jars[j].toString() + "' to Solr classloader");
}
return URLClassLoader.newInstance(jars, loader);
URL element = files[j].toURI().normalize().toURL();
log.info("Adding '" + element.toString() + "' to classloader");
elements[oldElements.length + j] = element;
} catch (MalformedURLException e) {
SolrException.log(log,"Can't construct solr lib class loader", e);
SolrException.log(log, "Can't add element to classloader: " + files[j], e);
}
}
log.info("Reusing parent classloader");
return loader;
return URLClassLoader.newInstance(elements, oldLoader.getParent());
}
// are we still here?
return oldLoader;
}
/**
* Convenience method for getting a new ClassLoader using all files found
* in the specified lib directory.
*/
static URLClassLoader createClassLoader(final File libDir, ClassLoader parent) {
if ( null == parent ) {
parent = Thread.currentThread().getContextClassLoader();
}
return replaceClassLoader(URLClassLoader.newInstance(new URL[0], parent),
libDir, null);
}
public SolrResourceLoader( String instanceDir )
@ -553,6 +615,4 @@ public class SolrResourceLoader implements ResourceLoader
}
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, builder.toString() );
}
}

View File

@ -30,6 +30,7 @@ import org.w3c.dom.NodeList;
import javax.xml.xpath.XPathConstants;
import java.io.IOException;
import java.io.InputStream;
public class TestConfig extends AbstractSolrTestCase {
@ -42,6 +43,31 @@ public class TestConfig extends AbstractSolrTestCase {
return "solrconfig-termindex.xml";
}
public void testLib() throws IOException {
SolrResourceLoader loader = h.getCore().getResourceLoader();
InputStream data = null;
String[] expectedFiles = new String[] { "empty-file-main-lib.txt",
"empty-file-a1.txt",
"empty-file-a2.txt",
"empty-file-b1.txt",
"empty-file-b2.txt",
"empty-file-c1.txt" };
for (String f : expectedFiles) {
data = loader.openResource(f);
assertNotNull("Should have found file " + f, data);
data.close();
}
String[] unexpectedFiles = new String[] { "empty-file-c2.txt",
"empty-file-d2.txt" };
for (String f : unexpectedFiles) {
data = null;
try {
data = loader.openResource(f);
} catch (Exception e) { /* :NOOP: (un)expected */ }
assertNull("should not have been able to find " + f, data);
}
}
public void testJavaProperty() {
// property values defined in build.xml

View File

@ -0,0 +1,18 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
Items under this directory are used by TestConfig.testLibs()

View File

@ -0,0 +1 @@

View File

@ -0,0 +1 @@

View File

@ -0,0 +1 @@

View File

@ -0,0 +1 @@

View File

@ -0,0 +1 @@

View File

@ -0,0 +1 @@

View File

@ -0,0 +1 @@

View File

@ -0,0 +1 @@

View File

@ -24,6 +24,12 @@
<config>
<!-- see TestConfig.testLib() -->
<lib dir="../lib-dirs/a" />
<lib dir="../lib-dirs/b" regex="b." />
<lib dir="../lib-dirs/c" regex="c1" />
<lib path="../lib-dirs/d/d1/" />
<jmx />
<!-- Used to specify an alternate directory to hold all index data.

View File

@ -0,0 +1,18 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
Items under this directory are used by TestConfig.testLibs()

View File

@ -0,0 +1 @@