LUCENE-4259: Allow reloading of codec/postings format list when classpath changes

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1366115 13f79535-47bb-0310-9956-ffa450edef68

lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java

@@ -33,8 +33,9 @@ import org.apache.lucene.util.SPIClassIterator;
  */
 public final class AnalysisSPILoader<S extends AbstractAnalysisFactory> {
 
-  private final Map<String,Class<? extends S>> services;
+  private volatile Map<String,Class<? extends S>> services = Collections.emptyMap();
   private final Class<S> clazz;
+  private final String[] suffixes;
   
   public AnalysisSPILoader(Class<S> clazz) {
     this(clazz, new String[] { clazz.getSimpleName() });
@@ -50,6 +51,22 @@ public final class AnalysisSPILoader<S extends AbstractAnalysisFactory> {
   
   public AnalysisSPILoader(Class<S> clazz, String[] suffixes, ClassLoader classloader) {
     this.clazz = clazz;
+    this.suffixes = suffixes;
+    reload(classloader);
+  }
+  
+  /** 
+   * Reloads the internal SPI list from the given {@link ClassLoader}.
+   * Changes to the service list are visible after the method ends, all
+   * iterators (e.g., from {@link #availableServices()},...) stay consistent. 
+   * 
+   * <p><b>NOTE:</b> Only new service providers are added, existing ones are
+   * never removed or replaced.
+   * 
+   * <p><em>This method is expensive and should only be called for discovery
+   * of new service providers on the given classpath/classloader!</em>
+   */
+  public void reload(ClassLoader classloader) {
     final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);
     final LinkedHashMap<String,Class<? extends S>> services = new LinkedHashMap<String,Class<? extends S>>();
     while (loader.hasNext()) {
@@ -69,6 +86,11 @@ public final class AnalysisSPILoader<S extends AbstractAnalysisFactory> {
       // only add the first one for each name, later services will be ignored
       // this allows to place services before others in classpath to make 
       // them used instead of others
+      //
+      // TODO: Should we disallow duplicate names here?
+      // Allowing it may get confusing on collisions, as different packages
+      // could contain same factory class, which is a naming bug!
+      // When changing this be careful to allow reload()!
       if (!services.containsKey(name)) {
         services.put(name, service);
       }

lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java

@@ -29,16 +29,7 @@ import org.apache.lucene.analysis.CharFilter;
 public abstract class CharFilterFactory extends AbstractAnalysisFactory {
 
   private static final AnalysisSPILoader<CharFilterFactory> loader =
-      getSPILoader(Thread.currentThread().getContextClassLoader());
+      new AnalysisSPILoader<CharFilterFactory>(CharFilterFactory.class);
-  
-  /**
-   * Used by e.g. Apache Solr to get a correctly configured instance
-   * of {@link AnalysisSPILoader} from Solr's classpath.
-   * @lucene.internal
-   */
-  public static AnalysisSPILoader<CharFilterFactory> getSPILoader(ClassLoader classloader) {
-    return new AnalysisSPILoader<CharFilterFactory>(CharFilterFactory.class, classloader);
-  }
   
   /** looks up a charfilter by name from context classpath */
   public static CharFilterFactory forName(String name) {
@@ -55,5 +46,21 @@ public abstract class CharFilterFactory extends AbstractAnalysisFactory {
     return loader.availableServices();
   }
 
+  /** 
+   * Reloads the factory list from the given {@link ClassLoader}.
+   * Changes to the factories are visible after the method ends, all
+   * iterators ({@link #availableCharFilters()},...) stay consistent. 
+   * 
+   * <p><b>NOTE:</b> Only new factories are added, existing ones are
+   * never removed or replaced.
+   * 
+   * <p><em>This method is expensive and should only be called for discovery
+   * of new factories on the given classpath/classloader!</em>
+   */
+  public static void reloadCharFilters(ClassLoader classloader) {
+    loader.reload(classloader);
+  }
+
+  /** Wraps the given Reader with a CharFilter. */
   public abstract Reader create(Reader input);
 }

lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java

@@ -28,17 +28,8 @@ import org.apache.lucene.analysis.TokenStream;
 public abstract class TokenFilterFactory extends AbstractAnalysisFactory {
 
   private static final AnalysisSPILoader<TokenFilterFactory> loader =
-      getSPILoader(Thread.currentThread().getContextClassLoader());
+      new AnalysisSPILoader<TokenFilterFactory>(TokenFilterFactory.class,
-  
+          new String[] { "TokenFilterFactory", "FilterFactory" });
-  /**
-   * Used by e.g. Apache Solr to get a correctly configured instance
-   * of {@link AnalysisSPILoader} from Solr's classpath.
-   * @lucene.internal
-   */
-  public static AnalysisSPILoader<TokenFilterFactory> getSPILoader(ClassLoader classloader) {
-    return new AnalysisSPILoader<TokenFilterFactory>(TokenFilterFactory.class,
-      new String[] { "TokenFilterFactory", "FilterFactory" }, classloader);
-  }
   
   /** looks up a tokenfilter by name from context classpath */
   public static TokenFilterFactory forName(String name) {
@@ -55,6 +46,21 @@ public abstract class TokenFilterFactory extends AbstractAnalysisFactory {
     return loader.availableServices();
   }
   
+  /** 
+   * Reloads the factory list from the given {@link ClassLoader}.
+   * Changes to the factories are visible after the method ends, all
+   * iterators ({@link #availableTokenFilters()},...) stay consistent. 
+   * 
+   * <p><b>NOTE:</b> Only new factories are added, existing ones are
+   * never removed or replaced.
+   * 
+   * <p><em>This method is expensive and should only be called for discovery
+   * of new factories on the given classpath/classloader!</em>
+   */
+  public static void reloadTokenFilters(ClassLoader classloader) {
+    loader.reload(classloader);
+  }
+
   /** Transform the specified input TokenStream */
   public abstract TokenStream create(TokenStream input);
 }

lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java

@@ -29,16 +29,7 @@ import java.util.Set;
 public abstract class TokenizerFactory extends AbstractAnalysisFactory {
 
   private static final AnalysisSPILoader<TokenizerFactory> loader =
-      getSPILoader(Thread.currentThread().getContextClassLoader());
+      new AnalysisSPILoader<TokenizerFactory>(TokenizerFactory.class);
-  
-  /**
-   * Used by e.g. Apache Solr to get a correctly configured instance
-   * of {@link AnalysisSPILoader} from Solr's classpath.
-   * @lucene.internal
-   */
-  public static AnalysisSPILoader<TokenizerFactory> getSPILoader(ClassLoader classloader) {
-    return new AnalysisSPILoader<TokenizerFactory>(TokenizerFactory.class, classloader);
-  }
   
   /** looks up a tokenizer by name from context classpath */
   public static TokenizerFactory forName(String name) {
@@ -55,6 +46,21 @@ public abstract class TokenizerFactory extends AbstractAnalysisFactory {
     return loader.availableServices();
   }
   
+  /** 
+   * Reloads the factory list from the given {@link ClassLoader}.
+   * Changes to the factories are visible after the method ends, all
+   * iterators ({@link #availableTokenizers()},...) stay consistent. 
+   * 
+   * <p><b>NOTE:</b> Only new factories are added, existing ones are
+   * never removed or replaced.
+   * 
+   * <p><em>This method is expensive and should only be called for discovery
+   * of new factories on the given classpath/classloader!</em>
+   */
+  public static void reloadTokenizers(ClassLoader classloader) {
+    loader.reload(classloader);
+  }
+
   /** Creates a TokenStream of the specified input */
   public abstract Tokenizer create(Reader input);
 }

lucene/core/src/java/org/apache/lucene/codecs/Codec.java

@@ -86,6 +86,21 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
     return loader.availableServices();
   }
   
+  /** 
+   * Reloads the codec list from the given {@link ClassLoader}.
+   * Changes to the codecs are visible after the method ends, all
+   * iterators ({@link #availableCodecs()},...) stay consistent. 
+   * 
+   * <p><b>NOTE:</b> Only new codecs are added, existing ones are
+   * never removed or replaced.
+   * 
+   * <p><em>This method is expensive and should only be called for discovery
+   * of new codecs on the given classpath/classloader!</em>
+   */
+  public static void reloadCodecs(ClassLoader classloader) {
+    loader.reload(classloader);
+  }
+  
   private static Codec defaultCodec = Codec.forName("Lucene40");
   
   /** expert: returns the default codec used for newly created

lucene/core/src/java/org/apache/lucene/codecs/PostingsFormat.java

@@ -70,4 +70,19 @@ public abstract class PostingsFormat implements NamedSPILoader.NamedSPI {
   public static Set<String> availablePostingsFormats() {
     return loader.availableServices();
   }
+  
+  /** 
+   * Reloads the postings format list from the given {@link ClassLoader}.
+   * Changes to the postings formats are visible after the method ends, all
+   * iterators ({@link #availablePostingsFormats()},...) stay consistent. 
+   * 
+   * <p><b>NOTE:</b> Only new postings formats are added, existing ones are
+   * never removed or replaced.
+   * 
+   * <p><em>This method is expensive and should only be called for discovery
+   * of new postings formats on the given classpath/classloader!</em>
+   */
+  public static void reloadPostingsFormats(ClassLoader classloader) {
+    loader.reload(classloader);
+  }
 }

lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java

@@ -28,16 +28,34 @@ import java.util.ServiceConfigurationError;
  * Helper class for loading named SPIs from classpath (e.g. Codec, PostingsFormat).
  * @lucene.internal
  */
-// TODO: would be nice to have case insensitive lookups.
 public final class NamedSPILoader<S extends NamedSPILoader.NamedSPI> implements Iterable<S> {
 
-  private final Map<String,S> services;
+  private volatile Map<String,S> services = Collections.emptyMap();
   private final Class<S> clazz;
 
   public NamedSPILoader(Class<S> clazz) {
+    this(clazz, Thread.currentThread().getContextClassLoader());
+  }
+  
+  public NamedSPILoader(Class<S> clazz, ClassLoader classloader) {
     this.clazz = clazz;
-    final SPIClassIterator<S> loader = SPIClassIterator.get(clazz);
+    reload(classloader);
-    final LinkedHashMap<String,S> services = new LinkedHashMap<String,S>();
+  }
+  
+  /** 
+   * Reloads the internal SPI list from the given {@link ClassLoader}.
+   * Changes to the service list are visible after the method ends, all
+   * iterators ({@link #iterator()},...) stay consistent. 
+   * 
+   * <p><b>NOTE:</b> Only new service providers are added, existing ones are
+   * never removed or replaced.
+   * 
+   * <p><em>This method is expensive and should only be called for discovery
+   * of new service providers on the given classpath/classloader!</em>
+   */
+  public void reload(ClassLoader classloader) {
+    final LinkedHashMap<String,S> services = new LinkedHashMap<String,S>(this.services);
+    final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);
     while (loader.hasNext()) {
       final Class<? extends S> c = loader.next();
       try {

solr/core/src/java/org/apache/solr/core/SolrConfig.java

@@ -436,9 +436,7 @@ public class SolrConfig extends Config {
    */
   public List<PluginInfo> getPluginInfos(String  type){
     List<PluginInfo> result = pluginStore.get(type);
-    return result == null ?
+    return result == null ? Collections.<PluginInfo>emptyList(): result; 
-            (List<PluginInfo>) Collections.EMPTY_LIST:
-            result; 
   }
   public PluginInfo getPluginInfo(String  type){
     List<PluginInfo> result = pluginStore.get(type);
@@ -446,14 +444,13 @@ public class SolrConfig extends Config {
   }
   
   private void initLibs() {
-    
     NodeList nodes = (NodeList) evaluate("lib", XPathConstants.NODESET);
-    if (nodes==null || nodes.getLength()==0)
+    if (nodes == null || nodes.getLength() == 0) return;
-      return;
     
     log.info("Adding specified lib dirs to ClassLoader");
     
-     for (int i=0; i<nodes.getLength(); i++) {
+    try {
+      for (int i = 0; i < nodes.getLength(); i++) {
         Node node = nodes.item(i);
         
         String baseDir = DOMUtil.getAttr(node, "dir");
@@ -466,9 +463,12 @@ public class SolrConfig extends Config {
         } else if (null != path) {
           getResourceLoader().addToClassLoader(path);
         } else {
-         throw new RuntimeException
+          throw new RuntimeException(
-           ("lib: missing mandatory attributes: 'dir' or 'path'");
+              "lib: missing mandatory attributes: 'dir' or 'path'");
         }
       }
+    } finally {
+      getResourceLoader().reloadLuceneSPI();
+    }
   }
 }

solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java

@@ -17,13 +17,11 @@
 
 package org.apache.solr.core;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -36,9 +34,9 @@ import org.apache.lucene.analysis.util.CharFilterFactory;
 import org.apache.lucene.analysis.util.ResourceLoaderAware;
 import org.apache.lucene.analysis.util.TokenFilterFactory;
 import org.apache.lucene.analysis.util.TokenizerFactory;
-import org.apache.lucene.analysis.util.AnalysisSPILoader;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.analysis.util.WordlistLoader;
-import org.apache.lucene.util.WeakIdentityMap;
 import org.apache.solr.common.ResourceLoader;
 import org.apache.solr.handler.admin.CoreAdminHandler;
 import org.apache.solr.handler.component.ShardHandlerFactory;
@@ -47,7 +45,6 @@ import org.slf4j.LoggerFactory;
 
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
-import java.nio.charset.CodingErrorAction;
 import java.lang.reflect.Constructor;
 
 import javax.naming.Context;
@@ -113,7 +110,7 @@ public class SolrResourceLoader implements ResourceLoader
     
     this.classLoader = createClassLoader(null, parent);
     addToClassLoader("./lib/", null);
-    
+    reloadLuceneSPI();
     this.coreProperties = coreProperties;
   }
 
@@ -134,7 +131,8 @@ public class SolrResourceLoader implements ResourceLoader
    * Adds every file/dir found in the baseDir which passes the specified Filter
    * to the ClassLoader used by this ResourceLoader.  This method <b>MUST</b>
    * only be called prior to using this ResourceLoader to get any resources, otherwise
-   * it's behavior will be non-deterministic.
+   * it's behavior will be non-deterministic. You also have to {link @reloadLuceneSPI}
+   * before using this ResourceLoader.
    *
    * @param baseDir base directory whose children (either jars or directories of
    *                classes) will be in the classpath, will be resolved relative
@@ -150,7 +148,8 @@ public class SolrResourceLoader implements ResourceLoader
    * Adds the specific file/dir specified to the ClassLoader used by this
    * ResourceLoader.  This method <b>MUST</b>
    * only be called prior to using this ResourceLoader to get any resources, otherwise
-   * it's behavior will be non-deterministic.
+   * it's behavior will be non-deterministic. You also have to {link #reloadLuceneSPI()}
+   * before using this ResourceLoader.
    *
    * @param path A jar file (or directory of classes) to be added to the classpath,
    *             will be resolved relative the instance dir.
@@ -169,6 +168,22 @@ public class SolrResourceLoader implements ResourceLoader
     }
   }
   
+  /**
+   * Reloads all Lucene SPI implementations using the new classloader.
+   * This method must be called after {@link #addToClassLoader(String)}
+   * and {@link #addToClassLoader(String,FileFilter)} before using
+   * this ResourceLoader.
+   */
+  void reloadLuceneSPI() {
+    // Codecs:
+    PostingsFormat.reloadPostingsFormats(this.classLoader);
+    Codec.reloadCodecs(this.classLoader);
+    // Analysis:
+    CharFilterFactory.reloadCharFilters(this.classLoader);
+    TokenFilterFactory.reloadTokenFilters(this.classLoader);
+    TokenizerFactory.reloadTokenizers(this.classLoader);
+  }
+  
   private static URLClassLoader replaceClassLoader(final URLClassLoader oldLoader,
                                                    final File base,
                                                    final FileFilter filter) {
@@ -351,9 +366,6 @@ public class SolrResourceLoader implements ResourceLoader
    */
   private static final Map<String, String> classNameCache = new ConcurrentHashMap<String, String>();
 
-  // A static map of AnalysisSPILoaders, keyed by ClassLoader used (because it can change during Solr lifetime) and expected base class:
-  private static final WeakIdentityMap<ClassLoader, Map<Class<?>,AnalysisSPILoader<?>>> expectedTypesSPILoaders = WeakIdentityMap.newConcurrentHashMap();
-
   // Using this pattern, legacy analysis components from previous Solr versions are identified and delegated to SPI loader:
   private static final Pattern legacyAnalysisPattern = 
       Pattern.compile("((\\Q"+base+".analysis.\\E)|(\\Q"+project+".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
@@ -388,26 +400,22 @@ public class SolrResourceLoader implements ResourceLoader
     // first try legacy analysis patterns, now replaced by Lucene's Analysis package:
     final Matcher m = legacyAnalysisPattern.matcher(cname);
     if (m.matches()) {
-      log.trace("Trying to load class from analysis SPI");
+      final String name = m.group(4);
-      // retrieve the map of classLoader -> expectedType -> SPI from cache / regenerate cache
+      log.trace("Trying to load class from analysis SPI using name='{}'", name);
-      Map<Class<?>,AnalysisSPILoader<?>> spiLoaders = expectedTypesSPILoaders.get(classLoader);
-      if (spiLoaders == null) {
-        spiLoaders = new IdentityHashMap<Class<?>,AnalysisSPILoader<?>>(3);
-        spiLoaders.put(CharFilterFactory.class, CharFilterFactory.getSPILoader(classLoader));
-        spiLoaders.put(TokenizerFactory.class, TokenizerFactory.getSPILoader(classLoader));
-        spiLoaders.put(TokenFilterFactory.class, TokenFilterFactory.getSPILoader(classLoader));
-        expectedTypesSPILoaders.put(classLoader, spiLoaders);
-      }
-      final AnalysisSPILoader<?> loader = spiLoaders.get(expectedType);
-      if (loader != null) {
-        // it's a correct expected type for analysis! Let's go on!
       try {
-          return clazz = loader.lookupClass(m.group(4)).asSubclass(expectedType);
+        if (CharFilterFactory.class == expectedType) {
+          return clazz = CharFilterFactory.lookupClass(name).asSubclass(expectedType);
+        } else if (TokenizerFactory.class == expectedType) {
+          return clazz = TokenizerFactory.lookupClass(name).asSubclass(expectedType);
+        } else if (TokenFilterFactory.class == expectedType) {
+          return clazz = TokenFilterFactory.lookupClass(name).asSubclass(expectedType);
+        } else {
+          log.warn("'{}' looks like an analysis factory, but caller requested different class type: {}", cname, expectedType.getName());
+        }
       } catch (IllegalArgumentException ex) { 
         // ok, we fall back to legacy loading
       }
     }
-    }
     
     // first try cname == full name
     try {