From 03d658a7bc306370cfce6ef92f34f151db7ad3dc Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Thu, 16 Jul 2020 16:05:24 +1000 Subject: [PATCH] SOLR-14151 Make schema components load from packages (#1669) --- solr/CHANGES.txt | 3 +- .../solr/api/CustomContainerPlugins.java | 12 +- .../java/org/apache/solr/core/ConfigSet.java | 26 ++- .../apache/solr/core/ConfigSetService.java | 8 +- .../org/apache/solr/core/CoreContainer.java | 16 +- .../java/org/apache/solr/core/PluginBag.java | 8 + .../java/org/apache/solr/core/PluginInfo.java | 50 ++++-- .../org/apache/solr/core/SolrClassLoader.java | 29 ++++ .../java/org/apache/solr/core/SolrConfig.java | 18 +++ .../java/org/apache/solr/core/SolrCore.java | 27 +++- .../apache/solr/core/SolrResourceLoader.java | 14 +- .../apache/solr/handler/SchemaHandler.java | 46 +++++- .../solr/handler/SolrConfigHandler.java | 2 +- .../apache/solr/handler/StreamHandler.java | 4 +- .../solr/handler/component/SearchHandler.java | 7 +- .../solr/packagemanager/PackageManager.java | 10 +- .../packagemanager/RepositoryManager.java | 6 +- .../java/org/apache/solr/pkg/PackageAPI.java | 11 +- .../org/apache/solr/pkg/PackageListeners.java | 51 ++++-- .../solr/pkg/PackageListeningClassLoader.java | 151 ++++++++++++++++++ .../org/apache/solr/pkg/PackageLoader.java | 5 + .../apache/solr/pkg/PackagePluginHolder.java | 33 ++-- .../solr/schema/FieldTypePluginLoader.java | 12 +- .../org/apache/solr/schema/IndexSchema.java | 109 +++++++------ .../solr/schema/ManagedIndexSchema.java | 2 +- .../apache/solr/schema/PreAnalyzedField.java | 2 +- .../util/plugin/AbstractPluginLoader.java | 8 +- .../runtimecode/schema-plugins.jar.bin | Bin 0 -> 6814 bytes solr/core/src/test-files/runtimecode/sig.txt | 5 + .../apache/solr/core/TestCodecSupport.java | 2 +- .../org/apache/solr/pkg/TestPackages.java | 140 +++++++++++++++- 31 files changed, 663 insertions(+), 154 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/core/SolrClassLoader.java create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java create mode 100644 solr/core/src/test-files/runtimecode/schema-plugins.jar.bin diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 375532f0469..53ca9e0fa2e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -108,7 +108,8 @@ Consult the lucene/CHANGES.txt file for additional, low level, changes in this r New Features --------------------- -(No changes) + +* SOLR-14151 Make schema components load from packages (noble) Improvements --------------------- diff --git a/solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java b/solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java index 6536276e294..f24626bbf08 100644 --- a/solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java +++ b/solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java @@ -36,12 +36,12 @@ import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.annotation.JsonProperty; import org.apache.solr.common.cloud.ClusterPropertiesListener; -import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.PathTrie; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.PluginInfo; import org.apache.solr.handler.admin.ContainerPluginsApi; import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrQueryRequest; @@ -225,12 +225,12 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri @SuppressWarnings({"unchecked","rawtypes"}) public ApiInfo(PluginMeta info, List errs) { this.info = info; - Pair klassInfo = org.apache.solr.core.PluginInfo.parseClassName(info.klass); - pkg = klassInfo.first(); + PluginInfo.ClassName klassInfo = new PluginInfo.ClassName(info.klass); + pkg = klassInfo.pkg; if (pkg != null) { PackageLoader.Package p = coreContainer.getPackageLoader().getPackage(pkg); if (p == null) { - errs.add("Invalid package " + klassInfo.first()); + errs.add("Invalid package " + klassInfo.pkg); return; } this.pkgVersion = p.getVersion(info.version); @@ -239,7 +239,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri return; } try { - klas = pkgVersion.getLoader().findClass(klassInfo.second(), Object.class); + klas = pkgVersion.getLoader().findClass(klassInfo.className, Object.class); } catch (Exception e) { log.error("Error loading class", e); errs.add("Error loading class " + e.toString()); @@ -247,7 +247,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri } } else { try { - klas = Class.forName(klassInfo.second()); + klas = Class.forName(klassInfo.className); } catch (ClassNotFoundException e) { errs.add("Error loading class " + e.toString()); return; diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSet.java b/solr/core/src/java/org/apache/solr/core/ConfigSet.java index d6cb31de460..81124772801 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSet.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSet.java @@ -19,6 +19,7 @@ package org.apache.solr.core; import org.apache.solr.common.util.NamedList; import org.apache.solr.schema.IndexSchema; + /** * Stores a core's configuration in the form of a SolrConfig and IndexSchema. * Immutable. @@ -30,7 +31,7 @@ public class ConfigSet { private final SolrConfig solrconfig; - private final IndexSchema indexSchema; + private final SchemaSupplier schemaSupplier; @SuppressWarnings({"rawtypes"}) private final NamedList properties; @@ -38,11 +39,11 @@ public class ConfigSet { private final boolean trusted; @SuppressWarnings({"rawtypes"}) - public ConfigSet(String name, SolrConfig solrConfig, IndexSchema indexSchema, + public ConfigSet(String name, SolrConfig solrConfig, SchemaSupplier indexSchemaSupplier, NamedList properties, boolean trusted) { this.name = name; this.solrconfig = solrConfig; - this.indexSchema = indexSchema; + this.schemaSupplier = indexSchemaSupplier; this.properties = properties; this.trusted = trusted; } @@ -55,8 +56,15 @@ public class ConfigSet { return solrconfig; } + /** + * + * @param forceFetch get a fresh value and not cached value + */ + public IndexSchema getIndexSchema(boolean forceFetch) { + return schemaSupplier.get(forceFetch); + } public IndexSchema getIndexSchema() { - return indexSchema; + return schemaSupplier.get(false); } @SuppressWarnings({"rawtypes"}) @@ -67,4 +75,14 @@ public class ConfigSet { public boolean isTrusted() { return trusted; } + + /**Provide a Schema object on demand + * We want IndexSchema Objects to be lazily instantiated because when a configset is + * created the {@link SolrResourceLoader} associated with it is not associated with a core + * So, we may not be able to update the core if we the schema classes are updated + * */ + interface SchemaSupplier { + IndexSchema get(boolean forceFetch); + + } } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index 84f94d5272b..22dacdf05d3 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -81,7 +81,7 @@ public abstract class ConfigSetService { ) ? false: true; SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted); - IndexSchema schema = createIndexSchema(dcore, solrConfig); + ConfigSet.SchemaSupplier schema = force -> createIndexSchema(dcore, solrConfig, force); return new ConfigSet(configSetName(dcore), solrConfig, schema, properties, trusted); } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, @@ -118,7 +118,7 @@ public abstract class ConfigSetService { * @param solrConfig the core's SolrConfig * @return an IndexSchema */ - protected IndexSchema createIndexSchema(CoreDescriptor cd, SolrConfig solrConfig) { + protected IndexSchema createIndexSchema(CoreDescriptor cd, SolrConfig solrConfig, boolean forceFetch) { // This is the schema name from the core descriptor. Sometimes users specify a custom schema file. // Important: indexSchemaFactory.create wants this! String cdSchemaName = cd.getSchemaName(); @@ -135,6 +135,7 @@ public abstract class ConfigSetService { if (modVersion != null) { // note: luceneMatchVersion influences the schema String cacheKey = configSet + "/" + guessSchemaName + "/" + modVersion + "/" + solrConfig.luceneMatchVersion; + if(forceFetch) schemaCache.invalidate(cacheKey); return schemaCache.get(cacheKey, (key) -> indexSchemaFactory.create(cdSchemaName, solrConfig)); } else { @@ -207,7 +208,8 @@ public abstract class ConfigSetService { @Override public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) { Path instanceDir = locateInstanceDir(cd); - return new SolrResourceLoader(instanceDir, parentLoader.getClassLoader()); + SolrResourceLoader solrResourceLoader = new SolrResourceLoader(instanceDir, parentLoader.getClassLoader()); + return solrResourceLoader; } @Override diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 53373b6d1d7..77983fd9b36 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -1588,20 +1589,33 @@ public class CoreContainer { return ret; } + /** + * reloads a core + * refer {@link CoreContainer#reload(String, UUID)} for details + */ + public void reload(String name) { + reload(name, null); + } /** * Recreates a SolrCore. * While the new core is loading, requests will continue to be dispatched to * and processed by the old core * * @param name the name of the SolrCore to reload + * @param coreId The unique Id of the core {@link SolrCore#uniqueId}. If this is null, it's reloaded anyway. If the current + * core has a different id, this is a no-op */ - public void reload(String name) { + public void reload(String name, UUID coreId) { if (isShutDown) { throw new AlreadyClosedException(); } SolrCore newCore = null; SolrCore core = solrCores.getCoreFromAnyList(name, false); if (core != null) { + if(coreId != null && core.uniqueId != coreId) { + //trying to reload an already unloaded core + return; + } // The underlying core properties files may have changed, we don't really know. So we have a (perhaps) stale // CoreDescriptor and we need to reload it from the disk files diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index 2f82ccc1cf2..d2442a130cc 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -368,6 +368,10 @@ public class PluginBag implements AutoCloseable { protected final PluginInfo pluginInfo; boolean registerAPI = false; + public PluginHolder(T inst, SolrConfig.SolrPluginInfo info) { + this.inst = inst; + pluginInfo = new PluginInfo(info.tag, Collections.singletonMap("class", inst.getClass().getName())); + } public PluginHolder(PluginInfo info) { this.pluginInfo = info; } @@ -413,6 +417,10 @@ public class PluginBag implements AutoCloseable { public PluginInfo getPluginInfo() { return pluginInfo; } + + public String toString() { + return String.valueOf(inst); + } } /** diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java index cc6615f5257..ae13fca4b85 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java +++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java @@ -25,7 +25,6 @@ import java.util.Map; import org.apache.solr.common.MapSerializable; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Pair; import org.apache.solr.util.DOMUtil; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -42,6 +41,7 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME; */ public class PluginInfo implements MapSerializable { public final String name, className, type, pkgName; + public final ClassName cName; @SuppressWarnings({"rawtypes"}) public final NamedList initArgs; public final Map attributes; @@ -53,9 +53,9 @@ public class PluginInfo implements MapSerializable { public PluginInfo(String type, Map attrs, @SuppressWarnings({"rawtypes"})NamedList initArgs, List children) { this.type = type; this.name = attrs.get(NAME); - Pair parsed = parseClassName(attrs.get(CLASS_NAME)); - this.className = parsed.second(); - this.pkgName = parsed.first(); + cName = parseClassName(attrs.get(CLASS_NAME)); + this.className = cName.className; + this.pkgName = cName.pkg; this.initArgs = initArgs; attributes = unmodifiableMap(attrs); this.children = children == null ? Collections.emptyList(): unmodifiableList(children); @@ -66,27 +66,45 @@ public class PluginInfo implements MapSerializable { * This checks if it is a package name prefixed classname. * the return value has first = package name and second = class name */ - public static Pair parseClassName(String name) { - String pkgName = null; - String className = name; - if (name != null) { + public static ClassName parseClassName(String name) { + return new ClassName(name); + } + + public static class ClassName { + public final String pkg; + public final String className; + public final String original; + + public ClassName(String name) { + this.original = name; + if (name == null) { + pkg = null; + className = null; + return; + } int colonIdx = name.indexOf(':'); if (colonIdx > -1) { - pkgName = name.substring(0, colonIdx); + pkg = name.substring(0, colonIdx); className = name.substring(colonIdx + 1); + } else { + pkg = null; + className = name; } } - return new Pair<>(pkgName, className); + @Override + public String toString() { + return original; + } } public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) { type = node.getNodeName(); name = DOMUtil.getAttr(node, NAME, requireName ? err : null); - Pair parsed = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null)); - className = parsed.second(); - pkgName = parsed.first(); + cName = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null)); + className = cName.className; + pkgName = cName.pkg; initArgs = DOMUtil.childNodesToNamedList(node); attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes())); children = loadSubPlugins(node); @@ -117,9 +135,9 @@ public class PluginInfo implements MapSerializable { } this.type = type; this.name = (String) m.get(NAME); - Pair parsed = parseClassName((String) m.get(CLASS_NAME)); - this.className = parsed.second(); - this.pkgName = parsed.first(); + cName = parseClassName((String) m.get(CLASS_NAME)); + this.className = cName.className; + this.pkgName = cName.pkg; attributes = unmodifiableMap(m); this.children = Collections.emptyList(); isFromSolrConfig = true; diff --git a/solr/core/src/java/org/apache/solr/core/SolrClassLoader.java b/solr/core/src/java/org/apache/solr/core/SolrClassLoader.java new file mode 100644 index 00000000000..7973b636486 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/SolrClassLoader.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core; + + +/** A generic interface to load plugin classes */ +public interface SolrClassLoader { + + T newInstance(String cname, Class expectedType, String... subpackages); + + @SuppressWarnings({"rawtypes"}) + T newInstance(String cName, Class expectedType, String[] subPackages, Class[] params, Object[] args); + + Class findClass(String cname, Class expectedType); +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 2daaa95ebeb..886179906f3 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -57,6 +57,8 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.IOUtils; import org.apache.solr.handler.component.SearchComponent; +import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.transform.TransformerFactory; @@ -106,6 +108,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { public static final String DEFAULT_CONF_FILE = "solrconfig.xml"; + private RequestParams requestParams; public enum PluginOpts { @@ -971,6 +974,21 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { return requestParams; } + /** + * The version of package that should be loaded for a given package name + * This information is stored in the params.json in the same configset + * If params.json is absent or there is no corresponding version specified for a given package, + * this returns a null and the latest is used by the caller + */ + public String maxPackageVersion(String pkg) { + RequestParams.ParamSet p = getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS); + if (p == null) { + return null; + } + Object o = p.get().get(pkg); + if (o == null || PackageLoader.LATEST.equals(o)) return null; + return o.toString(); + } public RequestParams refreshRequestParams() { requestParams = RequestParams.getFreshRequestParams(getResourceLoader(), requestParams); diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 0b1c551b054..4c9fae68e7a 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -47,6 +47,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -110,8 +111,7 @@ import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.metrics.SolrCoreMetricManager; import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.pkg.PackageListeners; -import org.apache.solr.pkg.PackageLoader; +import org.apache.solr.pkg.*; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.BinaryResponseWriter; @@ -191,6 +191,11 @@ public final class SolrCore implements SolrInfoBean, Closeable { private String name; private String logid; // used to show what name is set + /** + * A unique id to differentiate multiple instances of the same core + * If we reload a core, the name remains same , but the id will be new + */ + public final UUID uniqueId = UUID.randomUUID(); private boolean isReloaded = false; @@ -219,6 +224,8 @@ public final class SolrCore implements SolrInfoBean, Closeable { private IndexReaderFactory indexReaderFactory; private final Codec codec; private final MemClassLoader memClassLoader; + //singleton listener for all packages used in schema + private final PackageListeningClassLoader schemaPluginsLoader; private final CircuitBreakerManager circuitBreakerManager; @@ -271,6 +278,9 @@ public final class SolrCore implements SolrInfoBean, Closeable { public PackageListeners getPackageListeners() { return packageListeners; } + public PackageListeningClassLoader getSchemaPluginsLoader() { + return schemaPluginsLoader; + } static int boolean_query_max_clause_count = Integer.MIN_VALUE; @@ -885,6 +895,10 @@ public final class SolrCore implements SolrInfoBean, Closeable { public T createInitInstance(PluginInfo info, Class cast, String msg, String defClassName) { if (info == null) return null; T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName)); + return initPlugin(info, o); + } + + public static T initPlugin(PluginInfo info, T o) { if (o instanceof PluginInfoInitialized) { ((PluginInfoInitialized) o).init(info); } else if (o instanceof NamedListInitializedPlugin) { @@ -929,15 +943,20 @@ public final class SolrCore implements SolrInfoBean, Closeable { final CountDownLatch latch = new CountDownLatch(1); try { - IndexSchema schema = configSet.getIndexSchema(); CoreDescriptor cd = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null"); coreContainer.solrCores.addCoreDescriptor(cd); setName(name); - this.solrConfig = configSet.getSolrConfig(); this.resourceLoader = configSet.getSolrConfig().getResourceLoader(); + this.resourceLoader.core = this; + schemaPluginsLoader = new PackageListeningClassLoader(coreContainer, resourceLoader, + solrConfig::maxPackageVersion, + () -> setLatestSchema(configSet.getIndexSchema(true))); + this.packageListeners.addListener(schemaPluginsLoader); + IndexSchema schema = configSet.getIndexSchema(); + this.configSetProperties = configSet.getProperties(); // Initialize the metrics manager this.coreMetricManager = initCoreMetricManager(solrConfig); diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index 77c80bfa27e..4e7e15ccdb6 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; + import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.util.*; import org.apache.lucene.codecs.Codec; @@ -59,7 +60,7 @@ import org.slf4j.LoggerFactory; /** * @since solr 1.3 */ -public class SolrResourceLoader implements ResourceLoader, Closeable { +public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLoader { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String base = "org.apache.solr"; @@ -76,6 +77,12 @@ public class SolrResourceLoader implements ResourceLoader, Closeable { protected URLClassLoader classLoader; private final Path instanceDir; + /** + * this is set by the {@link SolrCore} + * This could be null if the core is not yet initialized + */ + SolrCore core; + private final List waitingForCore = Collections.synchronizedList(new ArrayList()); private final List infoMBeans = Collections.synchronizedList(new ArrayList()); private final List waitingForResources = Collections.synchronizedList(new ArrayList()); @@ -198,6 +205,11 @@ public class SolrResourceLoader implements ResourceLoader, Closeable { TokenizerFactory.reloadTokenizers(this.classLoader); } + public SolrCore getCore(){ + return core; + } + + private static URLClassLoader addURLsToClassLoader(final URLClassLoader oldLoader, List urls) { if (urls.size() == 0) { return oldLoader; diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index fb24b0d6434..0ab79e5f1c2 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -18,23 +18,21 @@ package org.apache.solr.handler; import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.function.BiConsumer; import org.apache.solr.api.Api; import org.apache.solr.api.ApiBag; import org.apache.solr.cloud.ZkSolrResourceLoader; +import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; @@ -181,6 +179,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, SimpleOrderedMap simpleOrderedMap = (SimpleOrderedMap) obj; if(name.equals(simpleOrderedMap.get("name"))) { rsp.add(fieldName.substring(0, realName.length() - 1), simpleOrderedMap); + insertPackageInfo(rsp.getValues(), req); return; } } @@ -190,6 +189,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, } else { rsp.add(fieldName, o); } + insertPackageInfo(rsp.getValues(), req); return; } @@ -202,6 +202,38 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, } } + /** + * If a plugin is loaded from a package, the version of the package being used should be added + * to the response + */ + @SuppressWarnings("rawtypes") + private void insertPackageInfo(Object o, SolrQueryRequest req) { + if (!req.getParams().getBool("meta", false)) return; + if (o instanceof List) { + List l = (List) o; + for (Object o1 : l) { + if (o1 instanceof NamedList || o1 instanceof List) insertPackageInfo(o1, req); + } + + } else if (o instanceof NamedList) { + NamedList nl = (NamedList) o; + nl.forEach((BiConsumer) (n, v) -> { + if (v instanceof NamedList || v instanceof List) insertPackageInfo(v, req); + }); + Object v = nl.get("class"); + if (v instanceof String) { + String klas = (String) v; + PluginInfo.ClassName parsedClassName = new PluginInfo.ClassName(klas); + if (parsedClassName.pkg != null) { + MapWriter mw = req.getCore().getSchemaPluginsLoader().getPackageVersion(parsedClassName); + if (mw != null) nl.add("_packageinfo_", mw); + } + } + + } + + } + private static Set subPaths = new HashSet<>(Arrays.asList( "version", "uniquekey", diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index 9ff0e66cde3..b94064ad6bd 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -267,7 +267,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa if (o instanceof Map) { @SuppressWarnings({"rawtypes"}) Map m1 = (Map) o; - m1.put("_packageinfo_", listener.getPackageVersion()); + m1.put("_packageinfo_", listener.getPackageVersion(info.cName)); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java index f1b15445dc2..d6046166cf8 100644 --- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java @@ -158,8 +158,8 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware, } @Override - protected void initNewInstance(PackageLoader.Package.Version newest) { - clazz = newest.getLoader().findClass(pluginInfo.className, Expressible.class); + protected Object initNewInstance(PackageLoader.Package.Version newest) { + return clazz = newest.getLoader().findClass(pluginInfo.className, Expressible.class); } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index b93d855d29b..c2bd1b68f4d 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -166,16 +166,11 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, } @Override - public void changed(PackageLoader.Package pkg) { + public void changed(PackageLoader.Package pkg, Ctx ctx) { //we could optimize this by listening to only relevant packages, // but it is not worth optimizing as these are lightweight objects components = null; } - - @Override - public PackageLoader.Package.Version getPackageVersion() { - return null; - } }); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index ef042dab93d..854bab11013 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -46,7 +46,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.packagemanager.SolrPackage.Command; import org.apache.solr.packagemanager.SolrPackage.Manifest; import org.apache.solr.packagemanager.SolrPackage.Plugin; -import org.apache.solr.pkg.PackagePluginHolder; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.util.SolrCLI; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -250,7 +250,7 @@ public class PackageManager implements Closeable { // Set the package version in the collection's parameters try { SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), - "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}"); + "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackageLoader.LATEST: packageInstance.version)+"'}}}"); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -296,7 +296,7 @@ public class PackageManager implements Closeable { // Set the package version in the collection's parameters try { SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), - "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}"); + "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackageLoader.LATEST: packageInstance.version) + "'}}}"); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -550,7 +550,7 @@ public class PackageManager implements Closeable { } } } - if (version == null || version.equalsIgnoreCase(PackageUtils.LATEST) || version.equalsIgnoreCase(PackagePluginHolder.LATEST)) { + if (version == null || version.equalsIgnoreCase(PackageUtils.LATEST) || version.equalsIgnoreCase(PackageLoader.LATEST)) { return latest; } else return null; } @@ -671,7 +671,7 @@ public class PackageManager implements Closeable { /** * Given a package, return a map of collections where this package is - * installed to the installed version (which can be {@link PackagePluginHolder#LATEST}) + * installed to the installed version (which can be {@link PackageLoader#LATEST}) */ public Map getDeployedCollections(String packageName) { List allCollections; diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 0d1fc177251..b5c371eb079 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -52,7 +52,7 @@ import org.apache.solr.filestore.PackageStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.pkg.PackageAPI; -import org.apache.solr.pkg.PackagePluginHolder; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.util.SolrCLI; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -300,7 +300,7 @@ public class RepositoryManager { /** * Install a version of the package. Also, run verify commands in case some - * collection was using {@link PackagePluginHolder#LATEST} version of this package and got auto-updated. + * collection was using {@link PackageLoader#LATEST} version of this package and got auto-updated. */ public boolean install(String packageName, String version) throws SolrException { SolrPackageRelease pkg = getLastPackageRelease(packageName); @@ -312,7 +312,7 @@ public class RepositoryManager { Map collectionsDeployedIn = packageManager.getDeployedCollections(packageName); List collectionsPeggedToLatest = collectionsDeployedIn.keySet().stream(). - filter(collection -> collectionsDeployedIn.get(collection).equals(PackagePluginHolder.LATEST)).collect(Collectors.toList()); + filter(collection -> collectionsDeployedIn.get(collection).equals(PackageLoader.LATEST)).collect(Collectors.toList()); if (!collectionsPeggedToLatest.isEmpty()) { PackageUtils.printGreen("Collections that will be affected (since they are configured to use $LATEST): "+collectionsPeggedToLatest); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index ecc344cfa1a..5a3e29a7120 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -192,7 +192,7 @@ public class PackageAPI { public PkgVersion(Package.AddVersion addVersion) { this.version = addVersion.version; - this.files = addVersion.files; + this.files = addVersion.files == null? null : Collections.unmodifiableList(addVersion.files); this.manifest = addVersion.manifest; this.manifestSHA512 = addVersion.manifestSHA512; } @@ -221,6 +221,15 @@ public class PackageAPI { throw new RuntimeException(e); } } + + public PkgVersion copy() { + PkgVersion result = new PkgVersion(); + result.version = this.version; + result.files = this.files; + result.manifest = this.manifest; + result.manifestSHA512 = this.manifestSHA512; + return result; + } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java index b5b295f156f..1895b6d3069 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -20,10 +20,11 @@ package org.apache.solr.pkg; import java.lang.invoke.MethodHandles; import java.lang.ref.Reference; import java.lang.ref.SoftReference; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; + +import org.apache.solr.common.MapWriter; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.logging.MDCLoggingContext; @@ -34,7 +35,7 @@ public class PackageListeners { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String PACKAGE_VERSIONS = "PKG_VERSIONS"; - private SolrCore core; + private final SolrCore core; public PackageListeners(SolrCore core) { this.core = core; @@ -64,21 +65,23 @@ public class PackageListeners { synchronized void packagesUpdated(List pkgs) { MDCLoggingContext.setCore(core); + Listener.Ctx ctx = new Listener.Ctx(); try { for (PackageLoader.Package pkgInfo : pkgs) { - invokeListeners(pkgInfo); + invokeListeners(pkgInfo, ctx); } } finally { + ctx.runLaterTasks(core::runAsync); MDCLoggingContext.clear(); } } - private synchronized void invokeListeners(PackageLoader.Package pkg) { + private synchronized void invokeListeners(PackageLoader.Package pkg, Listener.Ctx ctx) { for (Reference ref : listeners) { Listener listener = ref.get(); if(listener == null) continue; if (listener.packageName() == null || listener.packageName().equals(pkg.name())) { - listener.changed(pkg); + listener.changed(pkg, ctx); } } } @@ -96,15 +99,41 @@ public class PackageListeners { public interface Listener { - /**Name of the package or null to loisten to all package changes - */ + /**Name of the package or null to listen to all package changes */ String packageName(); PluginInfo pluginInfo(); - void changed(PackageLoader.Package pkg); + /**A callback when the package is updated */ + void changed(PackageLoader.Package pkg, Ctx ctx); - PackageLoader.Package.Version getPackageVersion(); + default MapWriter getPackageVersion(PluginInfo.ClassName cName) { + return null; + } + class Ctx { + private Map runLater; + + /** + * If there are multiple packages to be updated and there are multiple listeners, + * This is executed after all of the {@link Listener#changed(PackageLoader.Package, Ctx)} + * calls are invoked. The name is a unique identifier that can be used by consumers to avoid duplicate + * If no deduplication is required, use null as the name + */ + public void runLater(String name, Runnable runnable) { + if (runLater == null) runLater = new LinkedHashMap<>(); + if (name == null) { + name = runnable.getClass().getSimpleName() + "@" + runnable.hashCode(); + } + runLater.put(name, runnable); + } + + private void runLaterTasks(Consumer runnableExecutor) { + if (runLater == null) return; + for (Runnable r : runLater.values()) { + runnableExecutor.accept(r); + } + } + } } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java new file mode 100644 index 00000000000..ced4bd02936 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import org.apache.lucene.analysis.util.ResourceLoaderAware; +import org.apache.solr.common.MapWriter; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.SolrClassLoader; +import org.apache.solr.core.SolrResourceLoader; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +/** + * A {@link SolrClassLoader} that is designed to listen to a set of packages. + * This class registers a listener for each package that is loaded through this. + * If any of those packages are updated, the onReload runnable is run + * */ +public class PackageListeningClassLoader implements SolrClassLoader , PackageListeners.Listener { + private final CoreContainer coreContainer; + private final SolrResourceLoader coreResourceLoader; + private final Function pkgVersionSupplier; + /** package name and the versions that we are tracking + */ + private Map packageVersions = new HashMap<>(1); + private final Runnable onReload; + + public PackageListeningClassLoader(CoreContainer coreContainer, + SolrResourceLoader coreResourceLoader, + Function pkgVersionSupplier, + Runnable onReload) { + this.coreContainer = coreContainer; + this.coreResourceLoader = coreResourceLoader; + this.pkgVersionSupplier = pkgVersionSupplier; + this.onReload = () -> { + packageVersions = new HashMap<>(); + onReload.run(); + }; + } + + + @Override + public T newInstance(String cname, Class expectedType, String... subpackages) { + PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); + if(cName.pkg == null){ + return coreResourceLoader.newInstance(cname, expectedType, subpackages); + } else { + PackageLoader.Package.Version version = findPkgVersion(cName); + return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subpackages)); + + } + } + + private PackageLoader.Package.Version findPkgVersion(PluginInfo.ClassName cName) { + PackageLoader.Package.Version theVersion = coreContainer.getPackageLoader().getPackage(cName.pkg).getLatest(pkgVersionSupplier.apply(cName.pkg)); + packageVersions.put(cName.pkg, theVersion.getPkgVersion()); + return theVersion; + } + + @Override + public MapWriter getPackageVersion(PluginInfo.ClassName cName) { + if (cName.pkg == null) return null; + PackageAPI.PkgVersion p = packageVersions.get(cName.pkg); + return p == null ? null : p::writeMap; + } + + private T applyResourceLoaderAware(PackageLoader.Package.Version version, T obj) { + if (obj instanceof ResourceLoaderAware) { + SolrResourceLoader.assertAwareCompatibility(ResourceLoaderAware.class, obj); + try { + ((ResourceLoaderAware) obj).inform(version.getLoader()); + return obj; + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + } + return obj; + } + + @Override + @SuppressWarnings({"rawtypes"}) + public T newInstance(String cname, Class expectedType, String[] subPackages, Class[] params, Object[] args) { + PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); + if (cName.pkg == null) { + return coreResourceLoader.newInstance(cname, expectedType, subPackages, params, args); + } else { + PackageLoader.Package.Version version = findPkgVersion(cName); + return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args)); + } + } + + @Override + public Class findClass(String cname, Class expectedType) { + PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); + if (cName.pkg == null) { + return coreResourceLoader.findClass(cname, expectedType); + } else { + PackageLoader.Package.Version version = findPkgVersion(cName); + return version.getLoader().findClass(cName.className, expectedType); + + } + } + + @Override + public String packageName() { + return null; + } + + @Override + public PluginInfo pluginInfo() { + return null; + } + + @Override + public void changed(PackageLoader.Package pkg, Ctx ctx) { + PackageAPI.PkgVersion currVer = packageVersions.get(pkg.name); + if (currVer == null) { + //not watching this + return; + } + String latestSupportedVersion = pkgVersionSupplier.apply(pkg.name); + if (latestSupportedVersion == null) { + //no specific version configured. use the latest + latestSupportedVersion = pkg.getLatest().getVersion(); + } + if (Objects.equals(currVer.version, latestSupportedVersion)) { + //no need to update + return; + } + ctx.runLater(null, onReload); + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java index 562f8a05881..1e2fc574adb 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java @@ -50,6 +50,8 @@ import static org.apache.lucene.util.IOUtils.closeWhileHandlingException; */ public class PackageLoader implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String LATEST = "$LATEST"; + private final CoreContainer coreContainer; private final Map packageClassLoaders = new ConcurrentHashMap<>(); @@ -289,6 +291,9 @@ public class PackageLoader implements Closeable { public String getVersion() { return version.version; } + public PackageAPI.PkgVersion getPkgVersion(){ + return version.copy(); + } @SuppressWarnings({"rawtypes"}) public Collection getFiles() { diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index 322a1d3079d..0c6fd808d71 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -19,9 +19,16 @@ package org.apache.solr.pkg; import java.io.IOException; import java.lang.invoke.MethodHandles; + import org.apache.lucene.analysis.util.ResourceLoaderAware; +import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; -import org.apache.solr.core.*; +import org.apache.solr.core.PluginBag; +import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrInfoBean; +import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,32 +62,29 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { } @Override - public void changed(PackageLoader.Package pkg) { + public void changed(PackageLoader.Package pkg, Ctx ctx) { reload(pkg); } @Override - public PackageLoader.Package.Version getPackageVersion() { - return pkgVersion; + public MapWriter getPackageVersion(PluginInfo.ClassName cName) { + return pkgVersion == null ? null : ew -> pkgVersion.writeMap(ew); } }); } - private String maxVersion() { - RequestParams.ParamSet p = core.getSolrConfig().getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS); - if (p == null) { - return null; + public static PluginBag.PluginHolder createHolder(PluginInfo info, SolrCore core, Class type, String msg) { + if(info.cName.pkg == null) { + return new PluginBag.PluginHolder(info, core.createInitInstance(info, type,msg, null)); + } else { + return new PackagePluginHolder(info, core, SolrConfig.classVsSolrPluginInfo.get(type.getName())); } - Object o = p.get().get(info.pkgName); - if (o == null || LATEST.equals(o)) return null; - return o.toString(); } - private synchronized void reload(PackageLoader.Package pkg) { - String lessThan = maxVersion(); + String lessThan = core.getSolrConfig().maxPackageVersion(info.pkgName); PackageLoader.Package.Version newest = pkg.getLatest(lessThan); if (newest == null) { log.error("No latest version available for package : {}", pkg.name()); @@ -113,7 +117,7 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { } @SuppressWarnings({"unchecked"}) - protected void initNewInstance(PackageLoader.Package.Version newest) { + protected Object initNewInstance(PackageLoader.Package.Version newest) { Object instance = SolrCore.createInstance(pluginInfo.className, pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader()); PluginBag.initInstance(instance, pluginInfo); @@ -128,6 +132,7 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { log.error("error closing plugin", e); } } + return inst; } private void handleAwareCallbacks(SolrResourceLoader loader, Object instance) { diff --git a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java index f4b0b50e353..1f183264a3e 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java @@ -34,8 +34,8 @@ import org.apache.lucene.analysis.util.TokenizerFactory; import org.apache.lucene.util.Version; import org.apache.solr.analysis.TokenizerChain; import org.apache.solr.common.SolrException; +import org.apache.solr.core.SolrClassLoader; import org.apache.solr.core.SolrConfig; -import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.util.DOMUtil; import org.apache.solr.util.plugin.AbstractPluginLoader; import org.slf4j.Logger; @@ -78,7 +78,7 @@ public final class FieldTypePluginLoader @Override - protected FieldType create( SolrResourceLoader loader, + protected FieldType create( SolrClassLoader loader, String name, String className, Node node ) throws Exception { @@ -189,7 +189,7 @@ public final class FieldTypePluginLoader // private Analyzer readAnalyzer(Node node) throws XPathExpressionException { - final SolrResourceLoader loader = schema.getResourceLoader(); + final SolrClassLoader loader = schema.getSolrClassLoader(); // parent node used to be passed in as "fieldtype" // if (!fieldtype.hasChildNodes()) return null; @@ -255,7 +255,7 @@ public final class FieldTypePluginLoader @Override @SuppressWarnings({"rawtypes"}) - protected CharFilterFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { + protected CharFilterFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception { final Map params = DOMUtil.toMap(node.getAttributes()); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, CharFilterFactory.class.getSimpleName()).toString()); @@ -306,7 +306,7 @@ public final class FieldTypePluginLoader @Override @SuppressWarnings({"rawtypes"}) - protected TokenizerFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { + protected TokenizerFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception { final Map params = DOMUtil.toMap(node.getAttributes()); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenizerFactory.class.getSimpleName()).toString()); @@ -361,7 +361,7 @@ public final class FieldTypePluginLoader { @Override @SuppressWarnings({"rawtypes"}) - protected TokenFilterFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { + protected TokenFilterFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception { final Map params = DOMUtil.toMap(node.getAttributes()); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenFilterFactory.class.getSimpleName()).toString()); diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index bd1827d836f..af2a5b1e792 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -62,6 +62,7 @@ import org.apache.solr.common.util.Cache; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.SolrClassLoader; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.core.XmlConfigFile; @@ -135,6 +136,7 @@ public class IndexSchema { protected final Version luceneVersion; protected float version; protected final SolrResourceLoader loader; + protected final SolrClassLoader solrClassLoader; protected final Properties substitutableProperties; protected Map fields = new HashMap<>(); @@ -188,6 +190,7 @@ public class IndexSchema { protected IndexSchema(Version luceneVersion, SolrResourceLoader loader, Properties substitutableProperties) { this.luceneVersion = Objects.requireNonNull(luceneVersion); this.loader = loader; + this.solrClassLoader = loader.getCore() == null? loader: loader.getCore().getSchemaPluginsLoader(); this.substitutableProperties = substitutableProperties; } @@ -208,6 +211,10 @@ public class IndexSchema { return resourceName; } + public SolrClassLoader getSolrClassLoader() { + return solrClassLoader; + } + /** Sets the name of the resource used to instantiate this schema. */ public void setResourceName(String resourceName) { this.resourceName = resourceName; @@ -391,7 +398,7 @@ public class IndexSchema { return sf.getType().getUninversionType(sf); } // else... - + // It would be nice to throw a helpful error here, with a good useful message for the user, // but unfortunately, inspite of the UninvertingReader class jdoc claims that the uninversion // process is lazy, that doesn't mean it's lazy as of "When a caller attempts ot use doc values" @@ -405,7 +412,7 @@ public class IndexSchema { } /** - * Writes the schema in schema.xml format to the given writer + * Writes the schema in schema.xml format to the given writer */ void persist(Writer writer) throws IOException { final SolrQueryResponse response = new SolrQueryResponse(); @@ -499,18 +506,18 @@ public class IndexSchema { final FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, fieldTypes, schemaAware); expression = getFieldTypeXPathExpressions(); NodeList nodes = (NodeList) xpath.evaluate(expression, document, XPathConstants.NODESET); - typeLoader.load(loader, nodes); + typeLoader.load(solrClassLoader, nodes); // load the fields Map explicitRequiredProp = loadFields(document, xpath); expression = stepsToPath(SCHEMA, SIMILARITY); // /schema/similarity Node node = (Node) xpath.evaluate(expression, document, XPathConstants.NODE); - similarityFactory = readSimilarity(loader, node); + similarityFactory = readSimilarity(solrClassLoader, node); if (similarityFactory == null) { final Class simClass = SchemaSimilarityFactory.class; // use the loader to ensure proper SolrCoreAware handling - similarityFactory = loader.newInstance(simClass.getName(), SimilarityFactory.class); + similarityFactory = solrClassLoader.newInstance(simClass.getName(), SimilarityFactory.class); similarityFactory.init(new ModifiableSolrParams()); } else { isExplicitSimilarity = true; @@ -594,12 +601,12 @@ public class IndexSchema { uniqueKeyField.required = true; requiredFields.add(uniqueKeyField); } - } + } /////////////// parse out copyField commands /////////////// // Map> cfields = new HashMap>(); // expression = "/schema/copyField"; - + dynamicCopyFields = new DynamicCopy[] {}; loadCopyFields(document, xpath); @@ -754,10 +761,10 @@ public class IndexSchema { log.error(msg); throw new SolrException(ErrorCode.SERVER_ERROR, msg); } - + registerCopyField(source, dest, maxCharsInt); } - + for (Map.Entry entry : copyFieldTargetCounts.entrySet()) { if (entry.getValue() > 1 && !entry.getKey().multiValued()) { log.warn("Field {} is not multivalued and destination for multiople {} ({})" @@ -786,7 +793,7 @@ public class IndexSchema { } return false; } - + protected boolean isValidDynamicField(List dFields, SchemaField f) { String glob = f.getName(); if (f.getDefaultValue() != null) { @@ -853,7 +860,7 @@ public class IndexSchema { * inform( SolrCore core ) function for SolrCoreAware classes. * Outside inform, this could potentially throw a ConcurrentModificationException *

- * + * * @see SolrCoreAware */ public void registerCopyField(String source, String dest, int maxChars) { @@ -863,10 +870,10 @@ public class IndexSchema { DynamicField destDynamicField = null; SchemaField destSchemaField = fields.get(dest); SchemaField sourceSchemaField = fields.get(source); - + DynamicField sourceDynamicBase = null; DynamicField destDynamicBase = null; - + boolean sourceIsDynamicFieldReference = false; boolean sourceIsExplicitFieldGlob = false; @@ -892,7 +899,7 @@ public class IndexSchema { } } } - + if (null == destSchemaField || (null == sourceSchemaField && ! sourceIsExplicitFieldGlob)) { // Go through dynamicFields array only once, collecting info for both source and dest fields, if needed for (DynamicField dynamicField : dynamicFields) { @@ -914,7 +921,7 @@ public class IndexSchema { destDynamicBase = dynamicField; } } - if (null != destSchemaField + if (null != destSchemaField && (null != sourceSchemaField || sourceIsDynamicFieldReference || sourceIsExplicitFieldGlob)) { break; } @@ -971,7 +978,7 @@ public class IndexSchema { copyFieldList.add(new CopyField(sourceSchemaField, destSchemaField, maxChars)); incrementCopyFieldTargetCount(destSchemaField); } - + private void incrementCopyFieldTargetCount(SchemaField dest) { copyFieldTargetCounts.put(dest, copyFieldTargetCounts.containsKey(dest) ? copyFieldTargetCounts.get(dest) + 1 : 1); } @@ -983,7 +990,7 @@ public class IndexSchema { dynamicCopyFields = temp; } - static SimilarityFactory readSimilarity(SolrResourceLoader loader, Node node) { + static SimilarityFactory readSimilarity(SolrClassLoader loader, Node node) { if (node==null) { return null; } else { @@ -1024,7 +1031,7 @@ public class IndexSchema { else { return new NameEquals(regex); } } - + /** Returns true if the given name matches this pattern */ abstract boolean matches(String name); @@ -1033,7 +1040,7 @@ public class IndexSchema { /** Returns the result of combining this pattern's fixed string component with the given replacement */ abstract String subst(String replacement); - + /** Returns the length of the original regex, including the asterisk, if any. */ public int length() { return regex.length(); } @@ -1076,7 +1083,7 @@ public class IndexSchema { public int compareTo(DynamicReplacement other) { return other.pattern.length() - pattern.length(); } - + /** Returns the regex used to create this instance's pattern */ public String getRegex() { return pattern.regex; @@ -1110,7 +1117,7 @@ public class IndexSchema { public static class DynamicCopy extends DynamicReplacement { private final DynamicField destination; - + private final int maxChars; public int getMaxChars() { return maxChars; } @@ -1120,7 +1127,7 @@ public class IndexSchema { final DynamicField destDynamicBase; public DynamicField getDestDynamicBase() { return destDynamicBase; } - DynamicCopy(String sourceRegex, DynamicField destination, int maxChars, + DynamicCopy(String sourceRegex, DynamicField destination, int maxChars, DynamicField sourceDynamicBase, DynamicField destDynamicBase) { super(sourceRegex); this.destination = destination; @@ -1144,7 +1151,7 @@ public class IndexSchema { return destination.makeSchemaField(targetFieldName); } - + @Override public String toString() { return destination.prototype.toString(); @@ -1165,7 +1172,7 @@ public class IndexSchema { } return null; } - + /** * Does the schema explicitly define the specified field, i.e. not as a result * of a copyField declaration? We consider it explicitly defined if it matches @@ -1198,7 +1205,7 @@ public class IndexSchema { } return false; - } + } /** * Returns the SchemaField that should be used for the specified field name, or @@ -1373,10 +1380,10 @@ public class IndexSchema { return result; } - + /** - * Check if a field is used as the destination of a copyField operation - * + * Check if a field is used as the destination of a copyField operation + * * @since solr 1.3 */ public boolean isCopyFieldTarget( SchemaField f ) { @@ -1529,14 +1536,14 @@ public class IndexSchema { /** * Returns a list of copyField directives, with optional details and optionally restricting to those * directives that contain the requested source and/or destination field names. - * + * * @param showDetails If true, source and destination dynamic bases, and explicit fields matched by source globs, * will be added to dynamic copyField directives where appropriate * @param requestedSourceFields If not null, output is restricted to those copyField directives - * with the requested source field names + * with the requested source field names * @param requestedDestinationFields If not null, output is restricted to those copyField directives - * with the requested destination field names - * @return a list of copyField directives + * with the requested destination field names + * @return a list of copyField directives */ public List> getCopyFieldProperties (boolean showDetails, Set requestedSourceFields, Set requestedDestinationFields) { @@ -1613,7 +1620,7 @@ public class IndexSchema { * Requires synchronizing on the object returned by * {@link #getSchemaUpdateLock()}. * - * @param newField the SchemaField to add + * @param newField the SchemaField to add * @param persist to persist the schema or not * @return a new IndexSchema based on this schema with newField added * @see #newField(String, String, Map) @@ -1689,7 +1696,7 @@ public class IndexSchema { } /** - * Copies this schema, deletes the named field from the copy, creates a new field + * Copies this schema, deletes the named field from the copy, creates a new field * with the same name using the given args, then rebinds any referring copy fields * to the replacement field. * @@ -1699,7 +1706,7 @@ public class IndexSchema { * Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. * * @param fieldName The name of the field to be replaced - * @param replacementFieldType The field type of the replacement field + * @param replacementFieldType The field type of the replacement field * @param replacementArgs Initialization params for the replacement field * @return a new IndexSchema based on this schema with the named field replaced */ @@ -1757,7 +1764,7 @@ public class IndexSchema { * Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. * * @param fieldNamePattern The glob for the dynamic field to be replaced - * @param replacementFieldType The field type of the replacement dynamic field + * @param replacementFieldType The field type of the replacement dynamic field * @param replacementArgs Initialization params for the replacement dynamic field * @return a new IndexSchema based on this schema with the named dynamic field replaced */ @@ -1787,10 +1794,10 @@ public class IndexSchema { /** * Copies this schema and adds the new copy fields to the copy. - * - * Requires synchronizing on the object returned by + * + * Requires synchronizing on the object returned by * {@link #getSchemaUpdateLock()} - * + * * @param source source field name * @param destinations collection of target field names * @param maxChars max number of characters to copy from the source to each @@ -1812,7 +1819,7 @@ public class IndexSchema { * Requires synchronizing on the object returned by * {@link #getSchemaUpdateLock()}. * - * @param copyFields Key is the name of the source field name, value is a collection of target field names. + * @param copyFields Key is the name of the source field name, value is a collection of target field names. * Each corresponding copy field directives must exist. * @return The new Schema with the copy fields deleted */ @@ -1824,8 +1831,8 @@ public class IndexSchema { /** - * Returns a SchemaField if the given fieldName does not already - * exist in this schema, and does not match any dynamic fields + * Returns a SchemaField if the given fieldName does not already + * exist in this schema, and does not match any dynamic fields * in this schema. The resulting SchemaField can be used in a call * to {@link #addField(SchemaField)}. * @@ -1842,8 +1849,8 @@ public class IndexSchema { } /** - * Returns a SchemaField if the given dynamic field glob does not already - * exist in this schema, and does not match any dynamic fields + * Returns a SchemaField if the given dynamic field glob does not already + * exist in this schema, and does not match any dynamic fields * in this schema. The resulting SchemaField can be used in a call * to {@link #addField(SchemaField)}. * @@ -1904,15 +1911,15 @@ public class IndexSchema { } /** - * Copies this schema, deletes the named field type from the copy, creates a new field type + * Copies this schema, deletes the named field type from the copy, creates a new field type * with the same name using the given args, rebuilds fields and dynamic fields of the given * type, then rebinds any referring copy fields to the rebuilt fields. - * + * *

* The schema will not be persisted. *

* Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. - * + * * @param typeName The name of the field type to be replaced * @param replacementClassName The class name of the replacement field type * @param replacementArgs Initialization params for the replacement field type @@ -1944,14 +1951,14 @@ public class IndexSchema { protected String getFieldTypeXPathExpressions() { // /schema/fieldtype | /schema/fieldType | /schema/types/fieldtype | /schema/types/fieldType String expression = stepsToPath(SCHEMA, FIELD_TYPE.toLowerCase(Locale.ROOT)) // backcompat(?) - + XPATH_OR + stepsToPath(SCHEMA, FIELD_TYPE) - + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE.toLowerCase(Locale.ROOT)) - + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE); + + XPATH_OR + stepsToPath(SCHEMA, FIELD_TYPE) + + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE.toLowerCase(Locale.ROOT)) + + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE); return expression; } /** - * Helper method that returns true if the {@link #ROOT_FIELD_NAME} uses the exact + * Helper method that returns true if the {@link #ROOT_FIELD_NAME} uses the exact * same 'type' as the {@link #getUniqueKeyField()} * * @lucene.internal diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java index 7f114d937a9..d44e9619cc2 100644 --- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java @@ -1301,7 +1301,7 @@ public final class ManagedIndexSchema extends IndexSchema { Map newFieldTypes = new HashMap<>(); List schemaAwareList = new ArrayList<>(); FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, newFieldTypes, schemaAwareList); - typeLoader.loadSingle(loader, FieldTypeXmlAdapter.toNode(options)); + typeLoader.loadSingle(solrClassLoader, FieldTypeXmlAdapter.toNode(options)); FieldType ft = newFieldTypes.get(typeName); if (!schemaAwareList.isEmpty()) schemaAware.addAll(schemaAwareList); diff --git a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java index 3ba172030c6..84265e2d97a 100644 --- a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java +++ b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java @@ -79,7 +79,7 @@ public class PreAnalyzedField extends TextField implements HasImplicitIndexAnaly parser = new SimplePreAnalyzedParser(); } else { try { - Class implClazz = schema.getResourceLoader().findClass(implName, PreAnalyzedParser.class); + Class implClazz = schema.getSolrClassLoader().findClass(implName, PreAnalyzedParser.class); Constructor c = implClazz.getConstructor(new Class[0]); parser = (PreAnalyzedParser) c.newInstance(new Object[0]); } catch (Exception e) { diff --git a/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java b/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java index c98f80600db..568338ac209 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java @@ -23,7 +23,7 @@ import java.util.Objects; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.core.SolrClassLoader; import org.apache.solr.util.DOMUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,7 +86,7 @@ public abstract class AbstractPluginLoader * @param node - the XML node defining this plugin */ @SuppressWarnings("unchecked") - protected T create( SolrResourceLoader loader, String name, String className, Node node ) throws Exception + protected T create(SolrClassLoader loader, String name, String className, Node node ) throws Exception { return loader.newInstance(className, pluginClassType, getDefaultPackages()); } @@ -135,7 +135,7 @@ public abstract class AbstractPluginLoader * If a default element is defined, it will be returned from this function. * */ - public T load( SolrResourceLoader loader, NodeList nodes ) + public T load(SolrClassLoader loader, NodeList nodes ) { List info = new ArrayList<>(); T defaultPlugin = null; @@ -225,7 +225,7 @@ public abstract class AbstractPluginLoader * The created class for the plugin will be returned from this function. * */ - public T loadSingle(SolrResourceLoader loader, Node node) { + public T loadSingle(SolrClassLoader loader, Node node) { List info = new ArrayList<>(); T plugin = null; diff --git a/solr/core/src/test-files/runtimecode/schema-plugins.jar.bin b/solr/core/src/test-files/runtimecode/schema-plugins.jar.bin new file mode 100644 index 0000000000000000000000000000000000000000..4effbbafeb69266161a3420c7035cb5caa67e917 GIT binary patch literal 6814 zcmb7JWl$WRxaNN@-)0|d9h9RdsxAOr>p1c%@xIKc;ZCs+n|8wLr% z26xzeyIcFdef8c}?dd<=UDfAD-@5nIJy%oZF*-3CCMG5t^Y=6jw7-Gt&)ylr?dE99 ztp?Gw@bUtCxM+jjoGh%svUV083ieKaZWJu6yj(pXTvkpNo}O8{&iX(_;=m^|LZNl9 zb*BtToE=ZJmkqnfLUcQG>w}Z*I0C6sM2ClLyy4dMltFKT_v<24q_erZGrKj zWD<8CK3nbL7fn8gSw0W9S7gCh;=`7>i4>Wm>kKZq35{u8O}H*;H_o0plrj{Pqz49H z1}!nAtleeOXE$eb5`jGqzkeaPCJ(Oyi zhL{Urp-NXVK}vJ*UB-7sHO|N=L;3e33~oN;g)*Cpu{#4X@L7p4mPp>w3Cj8{q_$^& zWw?^yRS{<+bC5HaTq4Bn`P5E4?Ec=QcpyN65UTy_IJZC9^?(Na#`PKxx^CIFBWIKi zo7%HBakkA|-Ein4P=a-u(3CP&E4yzBszMJoaIKap(#LN&5|=9`Z!%zJ{8p5D(@V?W zBG8%bk2G*Uf>sla2cqI!Hy_Aq?R6JGA&%XRQ(Y)oTM!EG>rj{}Mo%zz-j{T#npFW7 zS<3?xs-sw5q<97w;SRsP}uLRdo&lp zEac$pU>sz@5~tFk z{>+{!Q87M_3RnL)CuH(!VDtHd;x0Vwd+aIvS5^79@gKBGs10e{h;`NbF)o^!Xo+1= zkGvoMz%Pk|0gB9S`>`9v2FU<*0=9j$EKi$TAc4*KM<=A0e+)nj;Mq4JWKeA|bXGU^ zK+5V#{iDFBYK30s`te8H4#X{fjpHkF!$oM;tC?=w0#c|E`F#Jc)k-WzdGXhxlMI!> zV{`=`Wr*t0)qeElGc&;8j$;jfz++fRS>E7e$?I*4svy={xXidrB=IU! zm<(jrC(RG^VF$Xqx{vf64!zzq(*wF9mzB zll6aPYMlX=xv?VTKG0s@)#r?1v4+v8H*)jkpdTXyIfoUwF_&UJuSY*vuP2n8Tb)zD z?1*ghZE-Xr)b{g4H+*A{Q!6k=SVS0MWjXj6@N|L_J4svHZ|viWu*5`*8n8XB!LZz0 ze_|*&HVwjmLVGeAG}2+_x7h@LG{+i`=D>>_2sG9kC7BktU2{Oo>9-a4e||UQTABv; zI~V8W{i-mFD}LDX!7uvm0R;~OhQ@uaaZOCJy9{AaNV%RZi2-wbjGwuv}E{0AcY2L-<@KprZ|b1cg}BPO+S!GjglYumi9rLOR(iz zq@7Jeu2|#Qm(IyV>Y{u0>pU!f9x5sOS3TI6^bC;8z)BCk?#xgRtkUYuhed`rh zmOjaRoLTc~((mG<1-;XW(+MPW)?(!}7|T>==oW>S(ELb-^vEkk!KtvQ#g9hpwH1Op zHMk*NQNNP0CD}FVvKPEK_tFHku5DflpQX5Kyi4M%X!N<)a_PNM?GxQ;gmU_tcd@&> z_S8L$dteqi4^w2|Zl}dJ`E+kK7d$I`)Kd+go6^(NDF6Jnx{OBICW}`+vqY(KLZU)l z(U?$dxa3ZO42hBx;!TNU0wNwYO_lWb>5SD7dx}xjNv20IP0{^O7LB8OlBFMmo=1}P z^;7}IHH^#oD!|J{+}a3P50gJC+$%jrKa$*)<4-!(EMXW>FG`54rP0%JV!1dHt6OG8 zz6QHRv88&^=*-)yza<%})uf+l=lB}RXwg?vrg>yu@F;EH4<0km`cCKclNfykS*;q* zPmI*&f~b*V^jP^l6O}^KI`MD&47#_`<4631vdpwK17BV$q&PQ=0>n1nX@K`m<0Pa> zwrV}zvSYIHam_U{)D5BLb)(wTvhg%UFQGries*2_y1;6mNE;u=esMN&!MEsnmlX{< zviru(OFCSbXFDa!;F86DSvGkX(ZsVK!wCPKy{&gQWX$nX(My`cAhIG^{sz0EQ?p?@ z`K#KVQM4@+Bppjk7Qqb5$744MPlN?D+mI2zW}eRXQ`_j?_?BjGaI~4`#IA0LI9Ir+ z8nRR3Kjaj5LWPV8GEVI^T_kUTtBYa=ropS+OebEwoGs93zlE5h@M5uo{c2D=bU2(J_4~oK|D% zA~dcj$Qz<1EA(Z8o(gZ?ddS9D0Q||pC3z8Dg2dW7=DXwiHN+xT(Zga@ek5*D4QohG zhJjAUuwXRKTc>_dPqw>oiEdG?38)36?HRD|DBRQZXPzFpIRZ#(M8%Im^Kvvj$F+D@ zl_!NaxtcN#%!eF_h`fR>K06IbcvP~|C?#ouIVfU*j|x;)Wbt+cHYF`AVD*=l&9L#g z#+q$Z*kt3+`$#)m^)i;Ex;3{Z}hx!ZMc$b zRT?36GNA8E>dR+n0Oh|a4}ru?Ts4%)(okm1SVPy zB(jNCro(~{todw=HY^-`PjX_w3Du>MBNac6lxANa9wAn~ewa7Nm;dDfO8zYyj*qvE zbm>a)O93Y+g6@(Yrlxw}@Bb zS6a+8)5p}kZ~I2Tb1i{pDZTaGPUs|%+*-a)>QaIYnr`!*Kut^YV_62fYE-IBvaM+P zY>nkgz?RAex>Il9)FV`Po&(uu@zTaI2}gou7>}h{fy2vlH9`<~HO-bj*>>1n)EF)U zL#if8X?y5zTY)aE`m&Qye(p)eu3gk)Lo^(U1EyCHexF&lgTb*MFqH4A zA6E8^{geRt;SFK>cUgFML(OLVfM#W?e&BBW{5z>x$K;Jp$feBlb7#CQk|_`ZGDT2t zwrzO>E?W!uMXzVtU0M%?&Trytz_!IGWk$0<8A!wkrjGx>i&EAx>RGtua)n)pjcCI3 zPwcVl>PO9&`3-~K{gf`VS^S80Zt?glE>VA0s2kZe*2Ie^m`I{WqB1CT!gdk(ZrpKr zWS559d0X!q8Okq!Tq@$kR})#g*he3pNZIz>b!<*z(1hR_4Dgehxx?XjH@_A*wbf=9 z+80m8d}8qZ&T2{O=aJ!5s2;?$-|+GIaHB~B)tse_*#hgcs^H;e*Dr0BYmDb0vf~EB zJZ5Gbm&uGtZ4{yUc7%_oPx1ndLxIJ}yxePezY=hkI(^CH%5%nTzlxMv@mnAvP)k3| zI8yq2D6u#FevHO2>Po`U%`977iXc$w1lXE{_E9;3}Q>^eivqF1RmNj1H zwIoVG@wY|HNfWt~-}$I!FM(2;@wz!Pb7Z&Yo=JM!z+(SGpK3PJ~I6MKW` zh+2!tMhV_fwZ#bX&xsZJe(Xo&5nrJXY*RF5Pgyb>&s@pw{XoKA#=`MemayK>(cp?Q z3lz~@zD+ODk6jti!h5I(p7iL}e`wm;#tM$rp5d8l%h>6x3}o~m&^>#xqs!!1awZmC z&)vyRwjd+0JC^I^yHGT~u+r&SD-~Oh&n?yP9*wG!UfhGo$GGo2o?>af%8K%pa+N2g z&*<8L?6D;pwqxTXjJ>4taXyOOV({DK>^|x(CW3`6Oc; zZv&|{b`Q~jy|eaLu`tm4y7mE=GV2>+eJSOkqH27&IlrX*tDuwyx5!I%eytG{Y2_NH z@8{Oa1i0tMv9d>76d9}h@{P1ZP1<+bUhZQdD8#ixuFateJEtccHJo%#?|R5PQR*KQ zS??yZIG}QgsrXH?LxxRwTs%rFr#0jTSKMVS?@6{Vg%7#N0~>!_?3KcUKVQcWlKYRS z@AcQ98WidNxXRP32heXIi|IzryaX5IU0`_nEofnR0Cf~lR_y(<6XX?B`!lyCMtE9h zZ;X576UoryB?AN#v#?&~kG`lPkLq=xT&>8}vcRGAhJxgRaC_tHW1T||64K8}qh`~6 z@$182ow3QrxR>q1Wbn4~)auI1XfyV)uRL}VA3qsx6%EzwaN3;2FwTh^N%!8^q($h< zBSHHDK-Q{A{i9&tv5p1`0u{WK?)&c^pAI@=s`eHZZ)yH&895*Eb?xJ!q3tmIUo9iu z-&;m~J9{s%r`sR9N7}{03E~I#_-~g-+eC#*nph&S?Cr7wvLVKd`<(8mtILp3nKkLP zw85El%LhizW=x=BlB{CoCock7h;-enVm}v@?`n;Fszl>*i(M%N|pxt zCFj_tU6!!6is)*!PvUq04n+nVsgcJ@<2pB2w4NNglp=B;!Ky}1qFV|_i>cXckZV8K53*%|-E{l+Fiz#_qu z0BBDzpa14lLMGn68EFidYzUJgsZoG43G)v50uhMKAu_5v^fHZjC_C~ z;(K)<-wqEaqbAp;<${MYcbo_73(jdxL(F`6B`TH(-*oDPuZd|qWG~JRI>zb(j=Snk z<#n~|R3H~!L2DWCYaHT>w4E%?<~>6R+e1?jcr>?pQ3VeluH~; zu!{1GJ^-}PV%W>rrCHR)mp+OD+P|Ba7T=Pp(?tAnYd8_PgZNH{DRTzGN(IjmtwAbW z7{WPw2qF3ns@+Z_zvUBKvN_3{cPa7OzYGzILY?6+az?@TzTN3WHVR*GIYtR7rqq^c zzEM77NA>%HbJ-PLdh|Xl)mwY%MEh^rTWFx{v51EaATAq?S5bqw4fZ4a~60@ib=Ald&@5+)U-SJ>U-z+?E6#zo!t67OVJ;@oWP@kQJ;OLU=)tV1)P)lI9HdL2frN~u zVzK5#<8R?LpaYEOxF%Fjii`#cX-24UmT?)a(;a4x_gcxP86o?-b;x3lBurCgc}ObG z5*H!xLsjTkgowJ`FKKSs?I(U3<*5jPBntSMPhmRZo#%&e8leMygMwi}P^iMX#h>ya*z8At6R z)6~MDNc8LK_C(ujjhra`Mr1_yd?WN})#kdL8$x+0+-p!kyxR)8pSTrmFTppsv;~$f zR$tSe=J|9X&>Blne3)IbeRtt)Nh5Te!kO@{F_})vTI7apcJPG~4La0hSfHSo%P2wE za*>p#t_-Z%rAR5+&y4%N^7rzmJAGQtu%g+f-y{>uoY<*|p9_%Fn%O4rH|=(_AXjlk zmNyIeub-OHuq-n&Ggj(`zRhh(pHqqmqf&@JXPVfhQ8Zz{l)GTi5}uy%1MEQ9oB6dX z8bJ?DtA>el(>phEKbj-uRhC#WfZ1r)k?S7@@NCt3mX*b@?1Of@wfbXNcdUaK{K14?X17t*R*G=XskIX znAGw5St)V*>l^X(qOEGM13qVUxnS?cHf8;E)qDJMxL;s5J``RZR4}j3w_9Q{8!`%v zqE?V9r-_*G`%wB^o}KF(FqLIMpaOyILid;eDI!n$D#Xd1(iY9kkH)s-8z4b&;Uo7) zhOCzmb{ROh1RF}+3fS8<&fQewQ$mQBCxouDM)K^#bfvw@EQ%YT%d2sL#tZjn+k3RN zwEfkUuk2Qr*$3@!a2I7{&Av7X*AL7Jg@vChlhIZJx-_WnhLZ5%R)@N=`L_5s?O6l& z@|%L%>vF0${2jb^ff?3Jv+h73;g2pR`zF+ULANchsuu=W=;bj1J`1^pnnOk*QF5M~ zh2|m=ErH$gDUMj7Otq3X8}(x19Z}Zm%iCU!s?`o#zH!f5(_|4Ti7?R6 zLb3m@&B8r{4@98Mf@*!_aEr|Q}O?Y`=7kte`fi+O8h1G x|G){+-&p>g{QooE-#_CoY551{fd7*2Uki<<3dWPao}&L*N`Ex$HN{`2{{TCk)Uf~n literal 0 HcmV?d00001 diff --git a/solr/core/src/test-files/runtimecode/sig.txt b/solr/core/src/test-files/runtimecode/sig.txt index 59beb47ca6e..2a42c7082c5 100644 --- a/solr/core/src/test-files/runtimecode/sig.txt +++ b/solr/core/src/test-files/runtimecode/sig.txt @@ -112,6 +112,11 @@ openssl dgst -sha512 expressible.jar.bin 3474a1414c8329c71ef5db2d3eb6e870363bdd7224a836aab561dccf5e8bcee4974ac799e72398c7e0b0c01972bab1c7454c8a4e791a8865bb676c0440627388 +openssl dgst -sha512 schema-plugins.jar.bin + +9299d8d5f8b358b66f113a0a91a0f55db09a41419872cb56e6e4ee402645487b75b17ec515f8035352e7fdcfac9e2e861f371d1e601869859384807b19abc2fb + + =============sha256============================ openssl dgst -sha256 runtimelibs.jar.bin diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java index 18a5f34d3ac..3ec43c9a086 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -210,7 +210,7 @@ public class TestCodecSupport extends SolrTestCaseJ4 { try { CoreDescriptor cd = new CoreDescriptor(newCoreName, testSolrHome.resolve(newCoreName), coreContainer); c = new SolrCore(coreContainer, cd, - new ConfigSet("fakeConfigset", config, schema, null, true)); + new ConfigSet("fakeConfigset", config, forceFetch -> schema, null, true)); assertNull(coreContainer.registerCore(cd, c, false, false)); h.coreName = newCoreName; assertEquals("We are not using the correct core", "solrconfig_codec2.xml", h.getCore().getConfigResource()); diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index d20efca7dcf..a54a3b09fe1 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -25,10 +25,7 @@ import java.util.concurrent.Callable; import org.apache.commons.codec.digest.DigestUtils; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoaderAware; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.*; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; @@ -39,6 +36,7 @@ import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.request.beans.Package; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; @@ -46,6 +44,7 @@ import org.apache.solr.common.MapWriterMap; import org.apache.solr.common.NavigableObject; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; @@ -642,6 +641,139 @@ public class TestPackages extends SolrCloudTestCase { } } + @SuppressWarnings("rawtypes") + public void testSchemaPlugins() throws Exception { + String COLLECTION_NAME = "testSchemaLoadingColl"; + System.setProperty("managed.schema.mutable", "true"); + + MiniSolrCloudCluster cluster = + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-managed")) + .configure(); + try { + String FILE1 = "/schemapkg/schema-plugins.jar"; + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster); + postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1, + "iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A=="); + + Package.AddVersion add = new Package.AddVersion(); + add.version = "1.0"; + add.pkg = "schemapkg"; + add.files = Arrays.asList(new String[]{FILE1}); + V2Request req = new V2Request.Builder("/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + req.process(cluster.getSolrClient()); + + TestDistribPackageStore.assertResponseValues(10, + () -> new V2Request.Builder("/cluster/package"). + withMethod(SolrRequest.METHOD.GET) + .build().process(cluster.getSolrClient()), + Utils.makeMap( + ":result:packages:schemapkg[0]:version", "1.0", + ":result:packages:schemapkg[0]:files[0]", FILE1 + )); + + CollectionAdminRequest + .createCollection(COLLECTION_NAME, "conf", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + String addFieldTypeAnalyzerWithClass = "{\n" + + "'add-field-type' : {" + + " 'name' : 'myNewTextFieldWithAnalyzerClass',\n" + + " 'class':'schemapkg:my.pkg.MyTextField',\n" + + " 'analyzer' : {\n" + + " 'luceneMatchVersion':'5.0.0'" ; +// + ",\n" + +// " 'class':'schemapkg:my.pkg.MyWhitespaceAnalyzer'\n"; + String charFilters = + " 'charFilters' : [{\n" + + " 'class':'schemapkg:my.pkg.MyPatternReplaceCharFilterFactory',\n" + + " 'replacement':'$1$1',\n" + + " 'pattern':'([a-zA-Z])\\\\\\\\1+'\n" + + " }],\n"; + String tokenizer = + " 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n"; + String filters = + " 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n"; + String suffix = " }\n" + + "}}"; + cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") { + + @Override + public RequestWriter.ContentWriter getContentWriter(String expectedType) { + return new RequestWriter.StringPayloadContentWriter(addFieldTypeAnalyzerWithClass + ',' + charFilters + tokenizer + filters + suffix, CommonParams.JSON_MIME); + } + + @Override + public SolrParams getParams() { + return null; + } + + @Override + public String getCollection() { + return COLLECTION_NAME; + } + + @Override + public SolrResponse createResponse(SolrClient client) { + return new SolrResponseBase(); + } + }); + verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass", + Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"1.0", + ":fieldType:analyzer:tokenizer:_packageinfo_:version","1.0", + ":fieldType:_packageinfo_:version","1.0")); + + add = new Package.AddVersion(); + add.version = "2.0"; + add.pkg = "schemapkg"; + add.files = Arrays.asList(new String[]{FILE1}); + req = new V2Request.Builder("/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + req.process(cluster.getSolrClient()); + + TestDistribPackageStore.assertResponseValues(10, + () -> new V2Request.Builder("/cluster/package"). + withMethod(SolrRequest.METHOD.GET) + .build().process(cluster.getSolrClient()), + Utils.makeMap( + ":result:packages:schemapkg[0]:version", "2.0", + ":result:packages:schemapkg[0]:files[0]", FILE1 + )); + + verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass", + Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"2.0", + ":fieldType:analyzer:tokenizer:_packageinfo_:version","2.0", + ":fieldType:_packageinfo_:version","2.0")); + + } finally { + cluster.shutdown(); + } + + } + @SuppressWarnings({"rawtypes","unchecked"}) + private void verifySchemaComponent(SolrClient client, String COLLECTION_NAME, String path, + Map expected) throws Exception { + SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, + WT, JAVABIN, + "meta", "true")); + + GenericSolrRequest req = new GenericSolrRequest(SolrRequest.METHOD.GET,path + , params); + TestDistribPackageStore.assertResponseValues(10, + client, + req, expected); + } + public static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { ByteBuffer fileContent = getFileContent(fname); String sha512 = DigestUtils.sha512Hex(fileContent.array());