SOLR-14281: Make sharedLib configurable through SysProp (#1288)

solr/CHANGES.txt

@@ -115,6 +115,9 @@ Improvements
 
 * SOLR-14286: Upgrade Jaegar to 1.1.0 (Cao Manh Dat)
 
+* SOLR-14281: Make sharedLib configurable through system property. Also allow multiple paths, separated by comma.
+  See Reference Guide 'format-of-solr-xml.adoc' and comment in 'solr.in.sh' for details (janhoy)
+
 Optimizations
 ---------------------
 

solr/bin/solr.in.cmd

@@ -192,7 +192,12 @@ REM set SOLR_OPTS="%SOLR_OPTS% -Dsolr.shardsWhitelist=http://localhost:8983,http
 REM For a visual indication in the Admin UI of what type of environment this cluster is, configure
 REM a -Dsolr.environment property below. Valid values are prod, stage, test, dev, with an optional
 REM label or color, e.g. -Dsolr.environment=test,label=Functional+test,color=brown
-REM SOLR_OPTS="$SOLR_OPTS -Dsolr.environment=prod"
+REM SOLR_OPTS="%SOLR_OPTS% -Dsolr.environment=prod"
+
+REM Specifies the path to a common library directory that will be shared across all cores.
+REM Any JAR files in this directory will be added to the search path for Solr plugins.
+REM If the specified path is not absolute, it will be relative to `%SOLR_HOME%`.
+REM SOLR_OPTS="%SOLR_OPTS% -Dsolr.sharedLib=/path/to/lib"
 
 REM Runs solr in a java security manager sandbox. This can protect against some attacks.
 REM Runtime properties are passed to the security policy file (server\etc\security.policy)

solr/bin/solr.in.sh

@@ -225,8 +225,14 @@
 # label or color, e.g. -Dsolr.environment=test,label=Functional+test,color=brown
 #SOLR_OPTS="$SOLR_OPTS -Dsolr.environment=prod"
 
+# Specifies the path to a common library directory that will be shared across all cores.
+# Any JAR files in this directory will be added to the search path for Solr plugins.
+# If the specified path is not absolute, it will be relative to `$SOLR_HOME`.
+#SOLR_OPTS="$SOLR_OPTS -Dsolr.sharedLib=/path/to/lib"
+
 # Runs solr in java security manager sandbox. This can protect against some attacks.
 # Runtime properties are passed to the security policy file (server/etc/security.policy)
 # You can also tweak via standard JDK files such as ~/.java.policy, see https://s.apache.org/java8policy
 # This is experimental! It may not work at all with Hadoop/HDFS features.
 #SOLR_SECURITY_MANAGER_ENABLED=true
+

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -29,11 +29,13 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -607,19 +609,31 @@ public class CoreContainer {
   public void load() {
     log.debug("Loading cores into CoreContainer [instanceDir={}]", loader.getInstancePath());
 
+    // Always add $SOLR_HOME/lib to the shared resource loader
+    Set<String> libDirs = new LinkedHashSet<>();
+    libDirs.add("lib");
+
+    if (!StringUtils.isBlank(cfg.getSharedLibDirectory())) {
+      List<String> sharedLibs = Arrays.asList(cfg.getSharedLibDirectory().split("\\s*,\\s*"));
+      libDirs.addAll(sharedLibs);
+    }
+
+    boolean modified = false;
     // add the sharedLib to the shared resource loader before initializing cfg based plugins
-    String libDir = cfg.getSharedLibDirectory();
+    for (String libDir : libDirs) {
-    if (libDir != null) {
       Path libPath = loader.getInstancePath().resolve(libDir);
       try {
         loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
-        loader.reloadLuceneSPI();
+        modified = true;
       } catch (IOException e) {
         if (!libDir.equals("lib")) { // Don't complain if default "lib" dir does not exist
           log.warn("Couldn't add files from {} to classpath: {}", libPath, e.getMessage());
         }
       }
     }
+    if (modified) {
+      loader.reloadLuceneSPI();
+    }
 
     packageStoreAPI = new PackageStoreAPI(this);
     containerHandlers.getApiBag().register(new AnnotatedApi(packageStoreAPI.readAPI), Collections.EMPTY_MAP);

solr/core/src/java/org/apache/solr/core/NodeConfig.java

@@ -161,6 +161,13 @@ public class NodeConfig {
     return replayUpdatesThreads;
   }
 
+  /**
+   * Returns a directory, optionally a comma separated list of directories
+   * that will be added to Solr's class path for searching for classes and plugins.
+   * The path is either absolute or relative to SOLR_HOME. Note that SOLR_HOME/lib
+   * will always be added to the search path even if not included in this list.
+   * @return a comma separated list of path strings or null if no paths defined
+   */
   public String getSharedLibDirectory() {
     return sharedLibDirectory;
   }
@@ -240,7 +247,7 @@ public class NodeConfig {
     private Path solrDataHome;
     private Integer booleanQueryMaxClauseCount;
     private Path configSetBaseDirectory;
-    private String sharedLibDirectory = "lib";
+    private String sharedLibDirectory;
     private PluginInfo shardHandlerFactoryConfig;
     private UpdateShardHandlerConfig updateShardHandlerConfig = UpdateShardHandlerConfig.DEFAULT;
     private String coreAdminHandlerClass = DEFAULT_ADMINHANDLERCLASS;

solr/core/src/test/org/apache/solr/core/TestCoreContainer.java

@@ -264,6 +264,14 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
       jar2.closeEntry();
     }
 
+    File customLib2 = new File(tmpRoot.toFile(), "customLib2");
+    customLib2.mkdirs();
+
+    try (JarOutputStream jar3 = new JarOutputStream(new FileOutputStream(new File(customLib2, "jar3.jar")))) {
+      jar3.putNextEntry(new JarEntry("jar3File"));
+      jar3.closeEntry();
+    }
+
     final CoreContainer cc1 = init(tmpRoot, "<solr></solr>");
     try {
       cc1.loader.openResource("defaultSharedLibFile").close();
@@ -271,6 +279,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
       cc1.shutdown();
     }
 
+    // Explicitly declaring 'lib' makes no change compared to the default
     final CoreContainer cc2 = init(tmpRoot, "<solr><str name=\"sharedLib\">lib</str></solr>");
     try {
       cc2.loader.openResource("defaultSharedLibFile").close();
@@ -278,12 +287,24 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
       cc2.shutdown();
     }
 
+    // custom lib folder, added to path in addition to default 'lib' folder
     final CoreContainer cc3 = init(tmpRoot, "<solr><str name=\"sharedLib\">customLib</str></solr>");
     try {
+      cc3.loader.openResource("defaultSharedLibFile").close();
       cc3.loader.openResource("customSharedLibFile").close();
     } finally {
       cc3.shutdown();
     }
+
+    // Comma separated list of lib folders
+    final CoreContainer cc4 = init(tmpRoot, "<solr><str name=\"sharedLib\">customLib, customLib2</str></solr>");
+    try {
+      cc4.loader.openResource("defaultSharedLibFile").close();
+      cc4.loader.openResource("customSharedLibFile").close();
+      cc4.loader.openResource("jar3File").close();
+    } finally {
+      cc4.shutdown();
+    }
   }
 
   private static final String CONFIGSETS_SOLR_XML ="<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" +

solr/server/solr/solr.xml

@@ -23,13 +23,14 @@
 
    More information about options available in this configuration file, 
    and Solr Core administration can be found online:
-   http://wiki.apache.org/solr/CoreAdmin
+   https://lucene.apache.org/solr/guide/format-of-solr-xml.html
 -->
 
 <solr>
 
   <int name="maxBooleanClauses">${solr.max.booleanClauses:1024}</int>
-  
+  <str name="sharedLib">${solr.sharedLib:}</str>
+
   <solrcloud>
 
     <str name="host">${host:}</str>

solr/solr-ref-guide/src/format-of-solr-xml.adoc

@@ -22,26 +22,32 @@ This section will describe the default `solr.xml` file included with Solr and ho
 
 == Defining solr.xml
 
-You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr`) in standalone mode or in ZooKeeper when using SolrCloud. The default `solr.xml` file looks like this:
+You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr` or `/var/solr/data`) or optionally in ZooKeeper when using SolrCloud. The default `solr.xml` file looks like this:
 
 [source,xml]
 ----
 <solr>
 
   <int name="maxBooleanClauses">${solr.max.booleanClauses:1024}</int>
+  <str name="sharedLib">${solr.sharedLib:}</str>
 
   <solrcloud>
     <str name="host">${host:}</str>
     <int name="hostPort">${jetty.port:8983}</int>
     <str name="hostContext">${hostContext:solr}</str>
-    <int name="zkClientTimeout">${zkClientTimeout:15000}</int>
     <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
+    <int name="zkClientTimeout">${zkClientTimeout:30000}</int>
+    <int name="distribUpdateSoTimeout">${distribUpdateSoTimeout:600000}</int>
+    <int name="distribUpdateConnTimeout">${distribUpdateConnTimeout:60000}</int>
+    <str name="zkCredentialsProvider">${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider}</str>
+    <str name="zkACLProvider">${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider}</str>
   </solrcloud>
 
   <shardHandlerFactory name="shardHandlerFactory"
     class="HttpShardHandlerFactory">
     <int name="socketTimeout">${socketTimeout:600000}</int>
-    <int name="connTimeout">${connTimeout:600000}</int>
+    <int name="connTimeout">${connTimeout:60000}</int>
+    <str name="shardsWhitelist">${solr.shardsWhitelist:}</str>
   </shardHandlerFactory>
 
 </solr>
@@ -83,7 +89,7 @@ The root of the core discovery tree, defaults to `$SOLR_HOME` (by default, `serv
 Currently non-operational.
 
 `sharedLib`::
-Specifies the path to a common library directory that will be shared across all cores. Any JAR files in this directory will be added to the search path for Solr plugins. This path is relative to `$SOLR_HOME`. Custom handlers may be placed in this directory.
+Specifies the path to a common library directory that will be shared across all cores. Any JAR files in this directory will be added to the search path for Solr plugins. If the specified path is not absolute, it will be relative to `$SOLR_HOME`. Custom handlers may be placed in this directory. Note that specifying `sharedLib` will not remove `$SOLR_HOME/lib` from Solr's class path.
 
 `shareSchema`::
 This attribute, when set to `true`, ensures that the multiple cores pointing to the same Schema resource file will be referring to the same IndexSchema Object. Sharing the IndexSchema Object makes loading the core faster. If you use this feature, make sure that no core-specific property is used in your Schema file.
@@ -203,6 +209,9 @@ If the threadpool uses a backing queue, what is its maximum size to use direct h
 `fairnessPolicy`::
 A boolean to configure if the threadpool favors fairness over throughput. Default is false to favor throughput.
 
+`shardsWhitelist`::
+When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can also be configured in `solr.in.sh`.
+
 `replicaRouting`::
 A NamedList specifying replica routing preference configuration. This may be used to select and configure replica routing preferences. `default=true` may be used to set the default base replica routing preference. Only positive default status assertions are respected; i.e., `default=false` has no effect. If no explicit default base replica routing preference is configured, the implicit default will be `random`.
 ----