From 62f5bd50cdd5ce8dd47267d39c02a0595f944064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 27 Feb 2020 14:16:33 +0100 Subject: [PATCH] SOLR-14281: Make sharedLib configurable through SysProp (#1288) --- solr/CHANGES.txt | 3 +++ solr/bin/solr.in.cmd | 7 ++++++- solr/bin/solr.in.sh | 6 ++++++ .../org/apache/solr/core/CoreContainer.java | 20 +++++++++++++++--- .../java/org/apache/solr/core/NodeConfig.java | 9 +++++++- .../apache/solr/core/TestCoreContainer.java | 21 +++++++++++++++++++ solr/server/solr/solr.xml | 5 +++-- .../src/format-of-solr-xml.adoc | 17 +++++++++++---- 8 files changed, 77 insertions(+), 11 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8137dd8ad85..8216bc98880 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -115,6 +115,9 @@ Improvements * SOLR-14286: Upgrade Jaegar to 1.1.0 (Cao Manh Dat) +* SOLR-14281: Make sharedLib configurable through system property. Also allow multiple paths, separated by comma. + See Reference Guide 'format-of-solr-xml.adoc' and comment in 'solr.in.sh' for details (janhoy) + Optimizations --------------------- diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd index cc85100170e..f018f3188d0 100755 --- a/solr/bin/solr.in.cmd +++ b/solr/bin/solr.in.cmd @@ -192,7 +192,12 @@ REM set SOLR_OPTS="%SOLR_OPTS% -Dsolr.shardsWhitelist=http://localhost:8983,http REM For a visual indication in the Admin UI of what type of environment this cluster is, configure REM a -Dsolr.environment property below. Valid values are prod, stage, test, dev, with an optional REM label or color, e.g. -Dsolr.environment=test,label=Functional+test,color=brown -REM SOLR_OPTS="$SOLR_OPTS -Dsolr.environment=prod" +REM SOLR_OPTS="%SOLR_OPTS% -Dsolr.environment=prod" + +REM Specifies the path to a common library directory that will be shared across all cores. +REM Any JAR files in this directory will be added to the search path for Solr plugins. +REM If the specified path is not absolute, it will be relative to `%SOLR_HOME%`. +REM SOLR_OPTS="%SOLR_OPTS% -Dsolr.sharedLib=/path/to/lib" REM Runs solr in a java security manager sandbox. This can protect against some attacks. REM Runtime properties are passed to the security policy file (server\etc\security.policy) diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh index 412c7a4573b..3b31cdc7843 100644 --- a/solr/bin/solr.in.sh +++ b/solr/bin/solr.in.sh @@ -225,8 +225,14 @@ # label or color, e.g. -Dsolr.environment=test,label=Functional+test,color=brown #SOLR_OPTS="$SOLR_OPTS -Dsolr.environment=prod" +# Specifies the path to a common library directory that will be shared across all cores. +# Any JAR files in this directory will be added to the search path for Solr plugins. +# If the specified path is not absolute, it will be relative to `$SOLR_HOME`. +#SOLR_OPTS="$SOLR_OPTS -Dsolr.sharedLib=/path/to/lib" + # Runs solr in java security manager sandbox. This can protect against some attacks. # Runtime properties are passed to the security policy file (server/etc/security.policy) # You can also tweak via standard JDK files such as ~/.java.policy, see https://s.apache.org/java8policy # This is experimental! It may not work at all with Hadoop/HDFS features. #SOLR_SECURITY_MANAGER_ENABLED=true + diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 9e5033ffaa9..901ff782c5b 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -29,11 +29,13 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -607,19 +609,31 @@ public class CoreContainer { public void load() { log.debug("Loading cores into CoreContainer [instanceDir={}]", loader.getInstancePath()); + // Always add $SOLR_HOME/lib to the shared resource loader + Set libDirs = new LinkedHashSet<>(); + libDirs.add("lib"); + + if (!StringUtils.isBlank(cfg.getSharedLibDirectory())) { + List sharedLibs = Arrays.asList(cfg.getSharedLibDirectory().split("\\s*,\\s*")); + libDirs.addAll(sharedLibs); + } + + boolean modified = false; // add the sharedLib to the shared resource loader before initializing cfg based plugins - String libDir = cfg.getSharedLibDirectory(); - if (libDir != null) { + for (String libDir : libDirs) { Path libPath = loader.getInstancePath().resolve(libDir); try { loader.addToClassLoader(SolrResourceLoader.getURLs(libPath)); - loader.reloadLuceneSPI(); + modified = true; } catch (IOException e) { if (!libDir.equals("lib")) { // Don't complain if default "lib" dir does not exist log.warn("Couldn't add files from {} to classpath: {}", libPath, e.getMessage()); } } } + if (modified) { + loader.reloadLuceneSPI(); + } packageStoreAPI = new PackageStoreAPI(this); containerHandlers.getApiBag().register(new AnnotatedApi(packageStoreAPI.readAPI), Collections.EMPTY_MAP); diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.java index 150b008e3c1..e03cdb12075 100644 --- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java +++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java @@ -161,6 +161,13 @@ public class NodeConfig { return replayUpdatesThreads; } + /** + * Returns a directory, optionally a comma separated list of directories + * that will be added to Solr's class path for searching for classes and plugins. + * The path is either absolute or relative to SOLR_HOME. Note that SOLR_HOME/lib + * will always be added to the search path even if not included in this list. + * @return a comma separated list of path strings or null if no paths defined + */ public String getSharedLibDirectory() { return sharedLibDirectory; } @@ -240,7 +247,7 @@ public class NodeConfig { private Path solrDataHome; private Integer booleanQueryMaxClauseCount; private Path configSetBaseDirectory; - private String sharedLibDirectory = "lib"; + private String sharedLibDirectory; private PluginInfo shardHandlerFactoryConfig; private UpdateShardHandlerConfig updateShardHandlerConfig = UpdateShardHandlerConfig.DEFAULT; private String coreAdminHandlerClass = DEFAULT_ADMINHANDLERCLASS; diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java index cb4ec021945..c6e2d8cc554 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java +++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java @@ -264,6 +264,14 @@ public class TestCoreContainer extends SolrTestCaseJ4 { jar2.closeEntry(); } + File customLib2 = new File(tmpRoot.toFile(), "customLib2"); + customLib2.mkdirs(); + + try (JarOutputStream jar3 = new JarOutputStream(new FileOutputStream(new File(customLib2, "jar3.jar")))) { + jar3.putNextEntry(new JarEntry("jar3File")); + jar3.closeEntry(); + } + final CoreContainer cc1 = init(tmpRoot, ""); try { cc1.loader.openResource("defaultSharedLibFile").close(); @@ -271,6 +279,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { cc1.shutdown(); } + // Explicitly declaring 'lib' makes no change compared to the default final CoreContainer cc2 = init(tmpRoot, "lib"); try { cc2.loader.openResource("defaultSharedLibFile").close(); @@ -278,12 +287,24 @@ public class TestCoreContainer extends SolrTestCaseJ4 { cc2.shutdown(); } + // custom lib folder, added to path in addition to default 'lib' folder final CoreContainer cc3 = init(tmpRoot, "customLib"); try { + cc3.loader.openResource("defaultSharedLibFile").close(); cc3.loader.openResource("customSharedLibFile").close(); } finally { cc3.shutdown(); } + + // Comma separated list of lib folders + final CoreContainer cc4 = init(tmpRoot, "customLib, customLib2"); + try { + cc4.loader.openResource("defaultSharedLibFile").close(); + cc4.loader.openResource("customSharedLibFile").close(); + cc4.loader.openResource("jar3File").close(); + } finally { + cc4.shutdown(); + } } private static final String CONFIGSETS_SOLR_XML ="\n" + diff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xml index 932f2a70712..7fce0e8b12a 100644 --- a/solr/server/solr/solr.xml +++ b/solr/server/solr/solr.xml @@ -23,13 +23,14 @@ More information about options available in this configuration file, and Solr Core administration can be found online: - http://wiki.apache.org/solr/CoreAdmin + https://lucene.apache.org/solr/guide/format-of-solr-xml.html --> ${solr.max.booleanClauses:1024} - + ${solr.sharedLib:} + ${host:} diff --git a/solr/solr-ref-guide/src/format-of-solr-xml.adoc b/solr/solr-ref-guide/src/format-of-solr-xml.adoc index 0190203e640..7d47a720459 100644 --- a/solr/solr-ref-guide/src/format-of-solr-xml.adoc +++ b/solr/solr-ref-guide/src/format-of-solr-xml.adoc @@ -22,26 +22,32 @@ This section will describe the default `solr.xml` file included with Solr and ho == Defining solr.xml -You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr`) in standalone mode or in ZooKeeper when using SolrCloud. The default `solr.xml` file looks like this: +You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr` or `/var/solr/data`) or optionally in ZooKeeper when using SolrCloud. The default `solr.xml` file looks like this: [source,xml] ---- ${solr.max.booleanClauses:1024} + ${solr.sharedLib:} ${host:} ${jetty.port:8983} ${hostContext:solr} - ${zkClientTimeout:15000} ${genericCoreNodeNames:true} + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} ${socketTimeout:600000} - ${connTimeout:600000} + ${connTimeout:60000} + ${solr.shardsWhitelist:} @@ -83,7 +89,7 @@ The root of the core discovery tree, defaults to `$SOLR_HOME` (by default, `serv Currently non-operational. `sharedLib`:: -Specifies the path to a common library directory that will be shared across all cores. Any JAR files in this directory will be added to the search path for Solr plugins. This path is relative to `$SOLR_HOME`. Custom handlers may be placed in this directory. +Specifies the path to a common library directory that will be shared across all cores. Any JAR files in this directory will be added to the search path for Solr plugins. If the specified path is not absolute, it will be relative to `$SOLR_HOME`. Custom handlers may be placed in this directory. Note that specifying `sharedLib` will not remove `$SOLR_HOME/lib` from Solr's class path. `shareSchema`:: This attribute, when set to `true`, ensures that the multiple cores pointing to the same Schema resource file will be referring to the same IndexSchema Object. Sharing the IndexSchema Object makes loading the core faster. If you use this feature, make sure that no core-specific property is used in your Schema file. @@ -203,6 +209,9 @@ If the threadpool uses a backing queue, what is its maximum size to use direct h `fairnessPolicy`:: A boolean to configure if the threadpool favors fairness over throughput. Default is false to favor throughput. +`shardsWhitelist`:: +When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can also be configured in `solr.in.sh`. + `replicaRouting`:: A NamedList specifying replica routing preference configuration. This may be used to select and configure replica routing preferences. `default=true` may be used to set the default base replica routing preference. Only positive default status assertions are respected; i.e., `default=false` has no effect. If no explicit default base replica routing preference is configured, the implicit default will be `random`. ----