Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4259: Allow reloading of codec/postings format list when classpath changes 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1366115 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler 2012-07-26 17:59:49 +00:00
parent 3d1279c06c
commit 69a6b5a562
9 changed files with 184 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java
View File

@ -33,8 +33,9 @@ import org.apache.lucene.util.SPIClassIterator;
*/
public final class AnalysisSPILoader<S extends AbstractAnalysisFactory> {
private final Map<String,Class<? extends S>> services;
private volatile Map<String,Class<? extends S>> services = Collections.emptyMap();
private final Class<S> clazz;
private final String[] suffixes;
public AnalysisSPILoader(Class<S> clazz) {
this(clazz, new String[] { clazz.getSimpleName() });
@ -50,6 +51,22 @@ public final class AnalysisSPILoader<S extends AbstractAnalysisFactory> {
public AnalysisSPILoader(Class<S> clazz, String[] suffixes, ClassLoader classloader) {
this.clazz = clazz;
this.suffixes = suffixes;
reload(classloader);
}
/**
* Reloads the internal SPI list from the given {@link ClassLoader}.
* Changes to the service list are visible after the method ends, all
* iterators (e.g., from {@link #availableServices()},...) stay consistent.
*
* <p><b>NOTE:</b> Only new service providers are added, existing ones are
* never removed or replaced.
*
* <p><em>This method is expensive and should only be called for discovery
* of new service providers on the given classpath/classloader!</em>
*/
public void reload(ClassLoader classloader) {
final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);
final LinkedHashMap<String,Class<? extends S>> services = new LinkedHashMap<String,Class<? extends S>>();
while (loader.hasNext()) {
@ -69,6 +86,11 @@ public final class AnalysisSPILoader<S extends AbstractAnalysisFactory> {
// only add the first one for each name, later services will be ignored
// this allows to place services before others in classpath to make
// them used instead of others
//
// TODO: Should we disallow duplicate names here?
// Allowing it may get confusing on collisions, as different packages
// could contain same factory class, which is a naming bug!
// When changing this be careful to allow reload()!
if (!services.containsKey(name)) {
services.put(name, service);
}

27
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java
View File

@ -29,16 +29,7 @@ import org.apache.lucene.analysis.CharFilter;
public abstract class CharFilterFactory extends AbstractAnalysisFactory {
private static final AnalysisSPILoader<CharFilterFactory> loader =
getSPILoader(Thread.currentThread().getContextClassLoader());
/**
* Used by e.g. Apache Solr to get a correctly configured instance
* of {@link AnalysisSPILoader} from Solr's classpath.
* @lucene.internal
*/
public static AnalysisSPILoader<CharFilterFactory> getSPILoader(ClassLoader classloader) {
return new AnalysisSPILoader<CharFilterFactory>(CharFilterFactory.class, classloader);
}
new AnalysisSPILoader<CharFilterFactory>(CharFilterFactory.class);
/** looks up a charfilter by name from context classpath */
public static CharFilterFactory forName(String name) {
@ -55,5 +46,21 @@ public abstract class CharFilterFactory extends AbstractAnalysisFactory {
return loader.availableServices();
}
/**
* Reloads the factory list from the given {@link ClassLoader}.
* Changes to the factories are visible after the method ends, all
* iterators ({@link #availableCharFilters()},...) stay consistent.
*
* <p><b>NOTE:</b> Only new factories are added, existing ones are
* never removed or replaced.
*
* <p><em>This method is expensive and should only be called for discovery
* of new factories on the given classpath/classloader!</em>
*/
public static void reloadCharFilters(ClassLoader classloader) {
loader.reload(classloader);
}
/** Wraps the given Reader with a CharFilter. */
public abstract Reader create(Reader input);
}

28
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java
View File

@ -28,17 +28,8 @@ import org.apache.lucene.analysis.TokenStream;
public abstract class TokenFilterFactory extends AbstractAnalysisFactory {
private static final AnalysisSPILoader<TokenFilterFactory> loader =
getSPILoader(Thread.currentThread().getContextClassLoader());
/**
* Used by e.g. Apache Solr to get a correctly configured instance
* of {@link AnalysisSPILoader} from Solr's classpath.
* @lucene.internal
*/
public static AnalysisSPILoader<TokenFilterFactory> getSPILoader(ClassLoader classloader) {
return new AnalysisSPILoader<TokenFilterFactory>(TokenFilterFactory.class,
new String[] { "TokenFilterFactory", "FilterFactory" }, classloader);
}
new AnalysisSPILoader<TokenFilterFactory>(TokenFilterFactory.class,
new String[] { "TokenFilterFactory", "FilterFactory" });
/** looks up a tokenfilter by name from context classpath */
public static TokenFilterFactory forName(String name) {
@ -55,6 +46,21 @@ public abstract class TokenFilterFactory extends AbstractAnalysisFactory {
return loader.availableServices();
}
/**
* Reloads the factory list from the given {@link ClassLoader}.
* Changes to the factories are visible after the method ends, all
* iterators ({@link #availableTokenFilters()},...) stay consistent.
*
* <p><b>NOTE:</b> Only new factories are added, existing ones are
* never removed or replaced.
*
* <p><em>This method is expensive and should only be called for discovery
* of new factories on the given classpath/classloader!</em>
*/
public static void reloadTokenFilters(ClassLoader classloader) {
loader.reload(classloader);
}
/** Transform the specified input TokenStream */
public abstract TokenStream create(TokenStream input);
}

26
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java
View File

@ -29,16 +29,7 @@ import java.util.Set;
public abstract class TokenizerFactory extends AbstractAnalysisFactory {
private static final AnalysisSPILoader<TokenizerFactory> loader =
getSPILoader(Thread.currentThread().getContextClassLoader());
/**
* Used by e.g. Apache Solr to get a correctly configured instance
* of {@link AnalysisSPILoader} from Solr's classpath.
* @lucene.internal
*/
public static AnalysisSPILoader<TokenizerFactory> getSPILoader(ClassLoader classloader) {
return new AnalysisSPILoader<TokenizerFactory>(TokenizerFactory.class, classloader);
}
new AnalysisSPILoader<TokenizerFactory>(TokenizerFactory.class);
/** looks up a tokenizer by name from context classpath */
public static TokenizerFactory forName(String name) {
@ -55,6 +46,21 @@ public abstract class TokenizerFactory extends AbstractAnalysisFactory {
return loader.availableServices();
}
/**
* Reloads the factory list from the given {@link ClassLoader}.
* Changes to the factories are visible after the method ends, all
* iterators ({@link #availableTokenizers()},...) stay consistent.
*
* <p><b>NOTE:</b> Only new factories are added, existing ones are
* never removed or replaced.
*
* <p><em>This method is expensive and should only be called for discovery
* of new factories on the given classpath/classloader!</em>
*/
public static void reloadTokenizers(ClassLoader classloader) {
loader.reload(classloader);
}
/** Creates a TokenStream of the specified input */
public abstract Tokenizer create(Reader input);
}

15
lucene/core/src/java/org/apache/lucene/codecs/Codec.java
View File

@ -86,6 +86,21 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
return loader.availableServices();
}
/**
* Reloads the codec list from the given {@link ClassLoader}.
* Changes to the codecs are visible after the method ends, all
* iterators ({@link #availableCodecs()},...) stay consistent.
*
* <p><b>NOTE:</b> Only new codecs are added, existing ones are
* never removed or replaced.
*
* <p><em>This method is expensive and should only be called for discovery
* of new codecs on the given classpath/classloader!</em>
*/
public static void reloadCodecs(ClassLoader classloader) {
loader.reload(classloader);
}
private static Codec defaultCodec = Codec.forName("Lucene40");
/** expert: returns the default codec used for newly created

15
lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java
View File

@ -70,4 +70,19 @@ public abstract class PostingsFormat implements NamedSPILoader.NamedSPI {
public static Set<String> availablePostingsFormats() {
return loader.availableServices();
}
/**
* Reloads the postings format list from the given {@link ClassLoader}.
* Changes to the postings formats are visible after the method ends, all
* iterators ({@link #availablePostingsFormats()},...) stay consistent.
*
* <p><b>NOTE:</b> Only new postings formats are added, existing ones are
* never removed or replaced.
*
* <p><em>This method is expensive and should only be called for discovery
* of new postings formats on the given classpath/classloader!</em>
*/
public static void reloadPostingsFormats(ClassLoader classloader) {
loader.reload(classloader);
}
}

26
lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java
View File

@ -28,16 +28,34 @@ import java.util.ServiceConfigurationError;
* Helper class for loading named SPIs from classpath (e.g. Codec, PostingsFormat).
* @lucene.internal
*/
// TODO: would be nice to have case insensitive lookups.
public final class NamedSPILoader<S extends NamedSPILoader.NamedSPI> implements Iterable<S> {
private final Map<String,S> services;
private volatile Map<String,S> services = Collections.emptyMap();
private final Class<S> clazz;
public NamedSPILoader(Class<S> clazz) {
this(clazz, Thread.currentThread().getContextClassLoader());
}
public NamedSPILoader(Class<S> clazz, ClassLoader classloader) {
this.clazz = clazz;
final SPIClassIterator<S> loader = SPIClassIterator.get(clazz);
final LinkedHashMap<String,S> services = new LinkedHashMap<String,S>();
reload(classloader);
}
/**
* Reloads the internal SPI list from the given {@link ClassLoader}.
* Changes to the service list are visible after the method ends, all
* iterators ({@link #iterator()},...) stay consistent.
*
* <p><b>NOTE:</b> Only new service providers are added, existing ones are
* never removed or replaced.
*
* <p><em>This method is expensive and should only be called for discovery
* of new service providers on the given classpath/classloader!</em>
*/
public void reload(ClassLoader classloader) {
final LinkedHashMap<String,S> services = new LinkedHashMap<String,S>(this.services);
final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);
while (loader.hasNext()) {
final Class<? extends S> c = loader.next();
try {

46
solr/core/src/java/org/apache/solr/core/SolrConfig.java
View File

@ -436,9 +436,7 @@ public class SolrConfig extends Config {
*/
public List<PluginInfo> getPluginInfos(String type){
List<PluginInfo> result = pluginStore.get(type);
return result == null ?
(List<PluginInfo>) Collections.EMPTY_LIST:
result;
return result == null ? Collections.<PluginInfo>emptyList(): result;
}
public PluginInfo getPluginInfo(String type){
List<PluginInfo> result = pluginStore.get(type);
@ -446,29 +444,31 @@ public class SolrConfig extends Config {
}
private void initLibs() {
NodeList nodes = (NodeList) evaluate("lib", XPathConstants.NODESET);
if (nodes==null || nodes.getLength()==0)
return;
if (nodes == null || nodes.getLength() == 0) return;
log.info("Adding specified lib dirs to ClassLoader");
for (int i=0; i<nodes.getLength(); i++) {
Node node = nodes.item(i);
String baseDir = DOMUtil.getAttr(node, "dir");
String path = DOMUtil.getAttr(node, "path");
if (null != baseDir) {
// :TODO: add support for a simpler 'glob' mutually eclusive of regex
String regex = DOMUtil.getAttr(node, "regex");
FileFilter filter = (null == regex) ? null : new RegexFileFilter(regex);
getResourceLoader().addToClassLoader(baseDir, filter);
} else if (null != path) {
getResourceLoader().addToClassLoader(path);
} else {
throw new RuntimeException
("lib: missing mandatory attributes: 'dir' or 'path'");
}
}
try {
for (int i = 0; i < nodes.getLength(); i++) {
Node node = nodes.item(i);
String baseDir = DOMUtil.getAttr(node, "dir");
String path = DOMUtil.getAttr(node, "path");
if (null != baseDir) {
// :TODO: add support for a simpler 'glob' mutually eclusive of regex
String regex = DOMUtil.getAttr(node, "regex");
FileFilter filter = (null == regex) ? null : new RegexFileFilter(regex);
getResourceLoader().addToClassLoader(baseDir, filter);
} else if (null != path) {
getResourceLoader().addToClassLoader(path);
} else {
throw new RuntimeException(
"lib: missing mandatory attributes: 'dir' or 'path'");
}
}
} finally {
getResourceLoader().reloadLuceneSPI();
}
}
}

64
solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
View File

@ -17,13 +17,11 @@
package org.apache.solr.core;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
@ -36,9 +34,9 @@ import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.analysis.util.AnalysisSPILoader;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.WeakIdentityMap;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
@ -47,7 +45,6 @@ import org.slf4j.LoggerFactory;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CodingErrorAction;
import java.lang.reflect.Constructor;
import javax.naming.Context;
@ -113,7 +110,7 @@ public class SolrResourceLoader implements ResourceLoader
this.classLoader = createClassLoader(null, parent);
addToClassLoader("./lib/", null);
reloadLuceneSPI();
this.coreProperties = coreProperties;
}
@ -134,7 +131,8 @@ public class SolrResourceLoader implements ResourceLoader
* Adds every file/dir found in the baseDir which passes the specified Filter
* to the ClassLoader used by this ResourceLoader. This method <b>MUST</b>
* only be called prior to using this ResourceLoader to get any resources, otherwise
* it's behavior will be non-deterministic.
* it's behavior will be non-deterministic. You also have to {link @reloadLuceneSPI}
* before using this ResourceLoader.
*
* @param baseDir base directory whose children (either jars or directories of
* classes) will be in the classpath, will be resolved relative
@ -150,7 +148,8 @@ public class SolrResourceLoader implements ResourceLoader
* Adds the specific file/dir specified to the ClassLoader used by this
* ResourceLoader. This method <b>MUST</b>
* only be called prior to using this ResourceLoader to get any resources, otherwise
* it's behavior will be non-deterministic.
* it's behavior will be non-deterministic. You also have to {link #reloadLuceneSPI()}
* before using this ResourceLoader.
*
* @param path A jar file (or directory of classes) to be added to the classpath,
* will be resolved relative the instance dir.
@ -169,6 +168,22 @@ public class SolrResourceLoader implements ResourceLoader
}
}
/**
* Reloads all Lucene SPI implementations using the new classloader.
* This method must be called after {@link #addToClassLoader(String)}
* and {@link #addToClassLoader(String,FileFilter)} before using
* this ResourceLoader.
*/
void reloadLuceneSPI() {
// Codecs:
PostingsFormat.reloadPostingsFormats(this.classLoader);
Codec.reloadCodecs(this.classLoader);
// Analysis:
CharFilterFactory.reloadCharFilters(this.classLoader);
TokenFilterFactory.reloadTokenFilters(this.classLoader);
TokenizerFactory.reloadTokenizers(this.classLoader);
}
private static URLClassLoader replaceClassLoader(final URLClassLoader oldLoader,
final File base,
final FileFilter filter) {
@ -351,9 +366,6 @@ public class SolrResourceLoader implements ResourceLoader
*/
private static final Map<String, String> classNameCache = new ConcurrentHashMap<String, String>();
// A static map of AnalysisSPILoaders, keyed by ClassLoader used (because it can change during Solr lifetime) and expected base class:
private static final WeakIdentityMap<ClassLoader, Map<Class<?>,AnalysisSPILoader<?>>> expectedTypesSPILoaders = WeakIdentityMap.newConcurrentHashMap();
// Using this pattern, legacy analysis components from previous Solr versions are identified and delegated to SPI loader:
private static final Pattern legacyAnalysisPattern =
Pattern.compile("((\\Q"+base+".analysis.\\E)|(\\Q"+project+".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
@ -388,24 +400,20 @@ public class SolrResourceLoader implements ResourceLoader
// first try legacy analysis patterns, now replaced by Lucene's Analysis package:
final Matcher m = legacyAnalysisPattern.matcher(cname);
if (m.matches()) {
log.trace("Trying to load class from analysis SPI");
// retrieve the map of classLoader -> expectedType -> SPI from cache / regenerate cache
Map<Class<?>,AnalysisSPILoader<?>> spiLoaders = expectedTypesSPILoaders.get(classLoader);
if (spiLoaders == null) {
spiLoaders = new IdentityHashMap<Class<?>,AnalysisSPILoader<?>>(3);
spiLoaders.put(CharFilterFactory.class, CharFilterFactory.getSPILoader(classLoader));
spiLoaders.put(TokenizerFactory.class, TokenizerFactory.getSPILoader(classLoader));
spiLoaders.put(TokenFilterFactory.class, TokenFilterFactory.getSPILoader(classLoader));
expectedTypesSPILoaders.put(classLoader, spiLoaders);
}
final AnalysisSPILoader<?> loader = spiLoaders.get(expectedType);
if (loader != null) {
// it's a correct expected type for analysis! Let's go on!
try {
return clazz = loader.lookupClass(m.group(4)).asSubclass(expectedType);
} catch (IllegalArgumentException ex) {
// ok, we fall back to legacy loading
final String name = m.group(4);
log.trace("Trying to load class from analysis SPI using name='{}'", name);
try {
if (CharFilterFactory.class == expectedType) {
return clazz = CharFilterFactory.lookupClass(name).asSubclass(expectedType);
} else if (TokenizerFactory.class == expectedType) {
return clazz = TokenizerFactory.lookupClass(name).asSubclass(expectedType);
} else if (TokenFilterFactory.class == expectedType) {
return clazz = TokenFilterFactory.lookupClass(name).asSubclass(expectedType);
} else {
log.warn("'{}' looks like an analysis factory, but caller requested different class type: {}", cname, expectedType.getName());
}
} catch (IllegalArgumentException ex) {
// ok, we fall back to legacy loading
}
}