diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 375532f0469..53ca9e0fa2e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -108,7 +108,8 @@ Consult the lucene/CHANGES.txt file for additional, low level, changes in this r New Features --------------------- -(No changes) + +* SOLR-14151 Make schema components load from packages (noble) Improvements --------------------- diff --git a/solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java b/solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java index 6536276e294..f24626bbf08 100644 --- a/solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java +++ b/solr/core/src/java/org/apache/solr/api/CustomContainerPlugins.java @@ -36,12 +36,12 @@ import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.annotation.JsonProperty; import org.apache.solr.common.cloud.ClusterPropertiesListener; -import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.PathTrie; import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.PluginInfo; import org.apache.solr.handler.admin.ContainerPluginsApi; import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrQueryRequest; @@ -225,12 +225,12 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri @SuppressWarnings({"unchecked","rawtypes"}) public ApiInfo(PluginMeta info, List errs) { this.info = info; - Pair klassInfo = org.apache.solr.core.PluginInfo.parseClassName(info.klass); - pkg = klassInfo.first(); + PluginInfo.ClassName klassInfo = new PluginInfo.ClassName(info.klass); + pkg = klassInfo.pkg; if (pkg != null) { PackageLoader.Package p = coreContainer.getPackageLoader().getPackage(pkg); if (p == null) { - errs.add("Invalid package " + klassInfo.first()); + errs.add("Invalid package " + klassInfo.pkg); return; } this.pkgVersion = p.getVersion(info.version); @@ -239,7 +239,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri return; } try { - klas = pkgVersion.getLoader().findClass(klassInfo.second(), Object.class); + klas = pkgVersion.getLoader().findClass(klassInfo.className, Object.class); } catch (Exception e) { log.error("Error loading class", e); errs.add("Error loading class " + e.toString()); @@ -247,7 +247,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri } } else { try { - klas = Class.forName(klassInfo.second()); + klas = Class.forName(klassInfo.className); } catch (ClassNotFoundException e) { errs.add("Error loading class " + e.toString()); return; diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSet.java b/solr/core/src/java/org/apache/solr/core/ConfigSet.java index d6cb31de460..81124772801 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSet.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSet.java @@ -19,6 +19,7 @@ package org.apache.solr.core; import org.apache.solr.common.util.NamedList; import org.apache.solr.schema.IndexSchema; + /** * Stores a core's configuration in the form of a SolrConfig and IndexSchema. * Immutable. @@ -30,7 +31,7 @@ public class ConfigSet { private final SolrConfig solrconfig; - private final IndexSchema indexSchema; + private final SchemaSupplier schemaSupplier; @SuppressWarnings({"rawtypes"}) private final NamedList properties; @@ -38,11 +39,11 @@ public class ConfigSet { private final boolean trusted; @SuppressWarnings({"rawtypes"}) - public ConfigSet(String name, SolrConfig solrConfig, IndexSchema indexSchema, + public ConfigSet(String name, SolrConfig solrConfig, SchemaSupplier indexSchemaSupplier, NamedList properties, boolean trusted) { this.name = name; this.solrconfig = solrConfig; - this.indexSchema = indexSchema; + this.schemaSupplier = indexSchemaSupplier; this.properties = properties; this.trusted = trusted; } @@ -55,8 +56,15 @@ public class ConfigSet { return solrconfig; } + /** + * + * @param forceFetch get a fresh value and not cached value + */ + public IndexSchema getIndexSchema(boolean forceFetch) { + return schemaSupplier.get(forceFetch); + } public IndexSchema getIndexSchema() { - return indexSchema; + return schemaSupplier.get(false); } @SuppressWarnings({"rawtypes"}) @@ -67,4 +75,14 @@ public class ConfigSet { public boolean isTrusted() { return trusted; } + + /**Provide a Schema object on demand + * We want IndexSchema Objects to be lazily instantiated because when a configset is + * created the {@link SolrResourceLoader} associated with it is not associated with a core + * So, we may not be able to update the core if we the schema classes are updated + * */ + interface SchemaSupplier { + IndexSchema get(boolean forceFetch); + + } } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index 84f94d5272b..22dacdf05d3 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -81,7 +81,7 @@ public abstract class ConfigSetService { ) ? false: true; SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted); - IndexSchema schema = createIndexSchema(dcore, solrConfig); + ConfigSet.SchemaSupplier schema = force -> createIndexSchema(dcore, solrConfig, force); return new ConfigSet(configSetName(dcore), solrConfig, schema, properties, trusted); } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, @@ -118,7 +118,7 @@ public abstract class ConfigSetService { * @param solrConfig the core's SolrConfig * @return an IndexSchema */ - protected IndexSchema createIndexSchema(CoreDescriptor cd, SolrConfig solrConfig) { + protected IndexSchema createIndexSchema(CoreDescriptor cd, SolrConfig solrConfig, boolean forceFetch) { // This is the schema name from the core descriptor. Sometimes users specify a custom schema file. // Important: indexSchemaFactory.create wants this! String cdSchemaName = cd.getSchemaName(); @@ -135,6 +135,7 @@ public abstract class ConfigSetService { if (modVersion != null) { // note: luceneMatchVersion influences the schema String cacheKey = configSet + "/" + guessSchemaName + "/" + modVersion + "/" + solrConfig.luceneMatchVersion; + if(forceFetch) schemaCache.invalidate(cacheKey); return schemaCache.get(cacheKey, (key) -> indexSchemaFactory.create(cdSchemaName, solrConfig)); } else { @@ -207,7 +208,8 @@ public abstract class ConfigSetService { @Override public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) { Path instanceDir = locateInstanceDir(cd); - return new SolrResourceLoader(instanceDir, parentLoader.getClassLoader()); + SolrResourceLoader solrResourceLoader = new SolrResourceLoader(instanceDir, parentLoader.getClassLoader()); + return solrResourceLoader; } @Override diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 53373b6d1d7..77983fd9b36 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -1588,20 +1589,33 @@ public class CoreContainer { return ret; } + /** + * reloads a core + * refer {@link CoreContainer#reload(String, UUID)} for details + */ + public void reload(String name) { + reload(name, null); + } /** * Recreates a SolrCore. * While the new core is loading, requests will continue to be dispatched to * and processed by the old core * * @param name the name of the SolrCore to reload + * @param coreId The unique Id of the core {@link SolrCore#uniqueId}. If this is null, it's reloaded anyway. If the current + * core has a different id, this is a no-op */ - public void reload(String name) { + public void reload(String name, UUID coreId) { if (isShutDown) { throw new AlreadyClosedException(); } SolrCore newCore = null; SolrCore core = solrCores.getCoreFromAnyList(name, false); if (core != null) { + if(coreId != null && core.uniqueId != coreId) { + //trying to reload an already unloaded core + return; + } // The underlying core properties files may have changed, we don't really know. So we have a (perhaps) stale // CoreDescriptor and we need to reload it from the disk files diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index 2f82ccc1cf2..d2442a130cc 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -368,6 +368,10 @@ public class PluginBag implements AutoCloseable { protected final PluginInfo pluginInfo; boolean registerAPI = false; + public PluginHolder(T inst, SolrConfig.SolrPluginInfo info) { + this.inst = inst; + pluginInfo = new PluginInfo(info.tag, Collections.singletonMap("class", inst.getClass().getName())); + } public PluginHolder(PluginInfo info) { this.pluginInfo = info; } @@ -413,6 +417,10 @@ public class PluginBag implements AutoCloseable { public PluginInfo getPluginInfo() { return pluginInfo; } + + public String toString() { + return String.valueOf(inst); + } } /** diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java index cc6615f5257..ae13fca4b85 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java +++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java @@ -25,7 +25,6 @@ import java.util.Map; import org.apache.solr.common.MapSerializable; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Pair; import org.apache.solr.util.DOMUtil; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -42,6 +41,7 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME; */ public class PluginInfo implements MapSerializable { public final String name, className, type, pkgName; + public final ClassName cName; @SuppressWarnings({"rawtypes"}) public final NamedList initArgs; public final Map attributes; @@ -53,9 +53,9 @@ public class PluginInfo implements MapSerializable { public PluginInfo(String type, Map attrs, @SuppressWarnings({"rawtypes"})NamedList initArgs, List children) { this.type = type; this.name = attrs.get(NAME); - Pair parsed = parseClassName(attrs.get(CLASS_NAME)); - this.className = parsed.second(); - this.pkgName = parsed.first(); + cName = parseClassName(attrs.get(CLASS_NAME)); + this.className = cName.className; + this.pkgName = cName.pkg; this.initArgs = initArgs; attributes = unmodifiableMap(attrs); this.children = children == null ? Collections.emptyList(): unmodifiableList(children); @@ -66,27 +66,45 @@ public class PluginInfo implements MapSerializable { * This checks if it is a package name prefixed classname. * the return value has first = package name and second = class name */ - public static Pair parseClassName(String name) { - String pkgName = null; - String className = name; - if (name != null) { + public static ClassName parseClassName(String name) { + return new ClassName(name); + } + + public static class ClassName { + public final String pkg; + public final String className; + public final String original; + + public ClassName(String name) { + this.original = name; + if (name == null) { + pkg = null; + className = null; + return; + } int colonIdx = name.indexOf(':'); if (colonIdx > -1) { - pkgName = name.substring(0, colonIdx); + pkg = name.substring(0, colonIdx); className = name.substring(colonIdx + 1); + } else { + pkg = null; + className = name; } } - return new Pair<>(pkgName, className); + @Override + public String toString() { + return original; + } } public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) { type = node.getNodeName(); name = DOMUtil.getAttr(node, NAME, requireName ? err : null); - Pair parsed = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null)); - className = parsed.second(); - pkgName = parsed.first(); + cName = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null)); + className = cName.className; + pkgName = cName.pkg; initArgs = DOMUtil.childNodesToNamedList(node); attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes())); children = loadSubPlugins(node); @@ -117,9 +135,9 @@ public class PluginInfo implements MapSerializable { } this.type = type; this.name = (String) m.get(NAME); - Pair parsed = parseClassName((String) m.get(CLASS_NAME)); - this.className = parsed.second(); - this.pkgName = parsed.first(); + cName = parseClassName((String) m.get(CLASS_NAME)); + this.className = cName.className; + this.pkgName = cName.pkg; attributes = unmodifiableMap(m); this.children = Collections.emptyList(); isFromSolrConfig = true; diff --git a/solr/core/src/java/org/apache/solr/core/SolrClassLoader.java b/solr/core/src/java/org/apache/solr/core/SolrClassLoader.java new file mode 100644 index 00000000000..7973b636486 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/SolrClassLoader.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core; + + +/** A generic interface to load plugin classes */ +public interface SolrClassLoader { + + T newInstance(String cname, Class expectedType, String... subpackages); + + @SuppressWarnings({"rawtypes"}) + T newInstance(String cName, Class expectedType, String[] subPackages, Class[] params, Object[] args); + + Class findClass(String cname, Class expectedType); +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 2daaa95ebeb..886179906f3 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -57,6 +57,8 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.IOUtils; import org.apache.solr.handler.component.SearchComponent; +import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.transform.TransformerFactory; @@ -106,6 +108,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { public static final String DEFAULT_CONF_FILE = "solrconfig.xml"; + private RequestParams requestParams; public enum PluginOpts { @@ -971,6 +974,21 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable { return requestParams; } + /** + * The version of package that should be loaded for a given package name + * This information is stored in the params.json in the same configset + * If params.json is absent or there is no corresponding version specified for a given package, + * this returns a null and the latest is used by the caller + */ + public String maxPackageVersion(String pkg) { + RequestParams.ParamSet p = getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS); + if (p == null) { + return null; + } + Object o = p.get().get(pkg); + if (o == null || PackageLoader.LATEST.equals(o)) return null; + return o.toString(); + } public RequestParams refreshRequestParams() { requestParams = RequestParams.getFreshRequestParams(getResourceLoader(), requestParams); diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 0b1c551b054..4c9fae68e7a 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -47,6 +47,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -110,8 +111,7 @@ import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.metrics.SolrCoreMetricManager; import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.pkg.PackageListeners; -import org.apache.solr.pkg.PackageLoader; +import org.apache.solr.pkg.*; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.BinaryResponseWriter; @@ -191,6 +191,11 @@ public final class SolrCore implements SolrInfoBean, Closeable { private String name; private String logid; // used to show what name is set + /** + * A unique id to differentiate multiple instances of the same core + * If we reload a core, the name remains same , but the id will be new + */ + public final UUID uniqueId = UUID.randomUUID(); private boolean isReloaded = false; @@ -219,6 +224,8 @@ public final class SolrCore implements SolrInfoBean, Closeable { private IndexReaderFactory indexReaderFactory; private final Codec codec; private final MemClassLoader memClassLoader; + //singleton listener for all packages used in schema + private final PackageListeningClassLoader schemaPluginsLoader; private final CircuitBreakerManager circuitBreakerManager; @@ -271,6 +278,9 @@ public final class SolrCore implements SolrInfoBean, Closeable { public PackageListeners getPackageListeners() { return packageListeners; } + public PackageListeningClassLoader getSchemaPluginsLoader() { + return schemaPluginsLoader; + } static int boolean_query_max_clause_count = Integer.MIN_VALUE; @@ -885,6 +895,10 @@ public final class SolrCore implements SolrInfoBean, Closeable { public T createInitInstance(PluginInfo info, Class cast, String msg, String defClassName) { if (info == null) return null; T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName)); + return initPlugin(info, o); + } + + public static T initPlugin(PluginInfo info, T o) { if (o instanceof PluginInfoInitialized) { ((PluginInfoInitialized) o).init(info); } else if (o instanceof NamedListInitializedPlugin) { @@ -929,15 +943,20 @@ public final class SolrCore implements SolrInfoBean, Closeable { final CountDownLatch latch = new CountDownLatch(1); try { - IndexSchema schema = configSet.getIndexSchema(); CoreDescriptor cd = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null"); coreContainer.solrCores.addCoreDescriptor(cd); setName(name); - this.solrConfig = configSet.getSolrConfig(); this.resourceLoader = configSet.getSolrConfig().getResourceLoader(); + this.resourceLoader.core = this; + schemaPluginsLoader = new PackageListeningClassLoader(coreContainer, resourceLoader, + solrConfig::maxPackageVersion, + () -> setLatestSchema(configSet.getIndexSchema(true))); + this.packageListeners.addListener(schemaPluginsLoader); + IndexSchema schema = configSet.getIndexSchema(); + this.configSetProperties = configSet.getProperties(); // Initialize the metrics manager this.coreMetricManager = initCoreMetricManager(solrConfig); diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index 77c80bfa27e..4e7e15ccdb6 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; + import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.util.*; import org.apache.lucene.codecs.Codec; @@ -59,7 +60,7 @@ import org.slf4j.LoggerFactory; /** * @since solr 1.3 */ -public class SolrResourceLoader implements ResourceLoader, Closeable { +public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLoader { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String base = "org.apache.solr"; @@ -76,6 +77,12 @@ public class SolrResourceLoader implements ResourceLoader, Closeable { protected URLClassLoader classLoader; private final Path instanceDir; + /** + * this is set by the {@link SolrCore} + * This could be null if the core is not yet initialized + */ + SolrCore core; + private final List waitingForCore = Collections.synchronizedList(new ArrayList()); private final List infoMBeans = Collections.synchronizedList(new ArrayList()); private final List waitingForResources = Collections.synchronizedList(new ArrayList()); @@ -198,6 +205,11 @@ public class SolrResourceLoader implements ResourceLoader, Closeable { TokenizerFactory.reloadTokenizers(this.classLoader); } + public SolrCore getCore(){ + return core; + } + + private static URLClassLoader addURLsToClassLoader(final URLClassLoader oldLoader, List urls) { if (urls.size() == 0) { return oldLoader; diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index fb24b0d6434..0ab79e5f1c2 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -18,23 +18,21 @@ package org.apache.solr.handler; import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.function.BiConsumer; import org.apache.solr.api.Api; import org.apache.solr.api.ApiBag; import org.apache.solr.cloud.ZkSolrResourceLoader; +import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; @@ -181,6 +179,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, SimpleOrderedMap simpleOrderedMap = (SimpleOrderedMap) obj; if(name.equals(simpleOrderedMap.get("name"))) { rsp.add(fieldName.substring(0, realName.length() - 1), simpleOrderedMap); + insertPackageInfo(rsp.getValues(), req); return; } } @@ -190,6 +189,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, } else { rsp.add(fieldName, o); } + insertPackageInfo(rsp.getValues(), req); return; } @@ -202,6 +202,38 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, } } + /** + * If a plugin is loaded from a package, the version of the package being used should be added + * to the response + */ + @SuppressWarnings("rawtypes") + private void insertPackageInfo(Object o, SolrQueryRequest req) { + if (!req.getParams().getBool("meta", false)) return; + if (o instanceof List) { + List l = (List) o; + for (Object o1 : l) { + if (o1 instanceof NamedList || o1 instanceof List) insertPackageInfo(o1, req); + } + + } else if (o instanceof NamedList) { + NamedList nl = (NamedList) o; + nl.forEach((BiConsumer) (n, v) -> { + if (v instanceof NamedList || v instanceof List) insertPackageInfo(v, req); + }); + Object v = nl.get("class"); + if (v instanceof String) { + String klas = (String) v; + PluginInfo.ClassName parsedClassName = new PluginInfo.ClassName(klas); + if (parsedClassName.pkg != null) { + MapWriter mw = req.getCore().getSchemaPluginsLoader().getPackageVersion(parsedClassName); + if (mw != null) nl.add("_packageinfo_", mw); + } + } + + } + + } + private static Set subPaths = new HashSet<>(Arrays.asList( "version", "uniquekey", diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index 9ff0e66cde3..b94064ad6bd 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -267,7 +267,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa if (o instanceof Map) { @SuppressWarnings({"rawtypes"}) Map m1 = (Map) o; - m1.put("_packageinfo_", listener.getPackageVersion()); + m1.put("_packageinfo_", listener.getPackageVersion(info.cName)); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java index f1b15445dc2..d6046166cf8 100644 --- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java @@ -158,8 +158,8 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware, } @Override - protected void initNewInstance(PackageLoader.Package.Version newest) { - clazz = newest.getLoader().findClass(pluginInfo.className, Expressible.class); + protected Object initNewInstance(PackageLoader.Package.Version newest) { + return clazz = newest.getLoader().findClass(pluginInfo.className, Expressible.class); } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index b93d855d29b..c2bd1b68f4d 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -166,16 +166,11 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, } @Override - public void changed(PackageLoader.Package pkg) { + public void changed(PackageLoader.Package pkg, Ctx ctx) { //we could optimize this by listening to only relevant packages, // but it is not worth optimizing as these are lightweight objects components = null; } - - @Override - public PackageLoader.Package.Version getPackageVersion() { - return null; - } }); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index ef042dab93d..854bab11013 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -46,7 +46,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.packagemanager.SolrPackage.Command; import org.apache.solr.packagemanager.SolrPackage.Manifest; import org.apache.solr.packagemanager.SolrPackage.Plugin; -import org.apache.solr.pkg.PackagePluginHolder; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.util.SolrCLI; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -250,7 +250,7 @@ public class PackageManager implements Closeable { // Set the package version in the collection's parameters try { SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), - "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}"); + "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackageLoader.LATEST: packageInstance.version)+"'}}}"); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -296,7 +296,7 @@ public class PackageManager implements Closeable { // Set the package version in the collection's parameters try { SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), - "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}"); + "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackageLoader.LATEST: packageInstance.version) + "'}}}"); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -550,7 +550,7 @@ public class PackageManager implements Closeable { } } } - if (version == null || version.equalsIgnoreCase(PackageUtils.LATEST) || version.equalsIgnoreCase(PackagePluginHolder.LATEST)) { + if (version == null || version.equalsIgnoreCase(PackageUtils.LATEST) || version.equalsIgnoreCase(PackageLoader.LATEST)) { return latest; } else return null; } @@ -671,7 +671,7 @@ public class PackageManager implements Closeable { /** * Given a package, return a map of collections where this package is - * installed to the installed version (which can be {@link PackagePluginHolder#LATEST}) + * installed to the installed version (which can be {@link PackageLoader#LATEST}) */ public Map getDeployedCollections(String packageName) { List allCollections; diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 0d1fc177251..b5c371eb079 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -52,7 +52,7 @@ import org.apache.solr.filestore.PackageStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.pkg.PackageAPI; -import org.apache.solr.pkg.PackagePluginHolder; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.util.SolrCLI; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -300,7 +300,7 @@ public class RepositoryManager { /** * Install a version of the package. Also, run verify commands in case some - * collection was using {@link PackagePluginHolder#LATEST} version of this package and got auto-updated. + * collection was using {@link PackageLoader#LATEST} version of this package and got auto-updated. */ public boolean install(String packageName, String version) throws SolrException { SolrPackageRelease pkg = getLastPackageRelease(packageName); @@ -312,7 +312,7 @@ public class RepositoryManager { Map collectionsDeployedIn = packageManager.getDeployedCollections(packageName); List collectionsPeggedToLatest = collectionsDeployedIn.keySet().stream(). - filter(collection -> collectionsDeployedIn.get(collection).equals(PackagePluginHolder.LATEST)).collect(Collectors.toList()); + filter(collection -> collectionsDeployedIn.get(collection).equals(PackageLoader.LATEST)).collect(Collectors.toList()); if (!collectionsPeggedToLatest.isEmpty()) { PackageUtils.printGreen("Collections that will be affected (since they are configured to use $LATEST): "+collectionsPeggedToLatest); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index ecc344cfa1a..5a3e29a7120 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -192,7 +192,7 @@ public class PackageAPI { public PkgVersion(Package.AddVersion addVersion) { this.version = addVersion.version; - this.files = addVersion.files; + this.files = addVersion.files == null? null : Collections.unmodifiableList(addVersion.files); this.manifest = addVersion.manifest; this.manifestSHA512 = addVersion.manifestSHA512; } @@ -221,6 +221,15 @@ public class PackageAPI { throw new RuntimeException(e); } } + + public PkgVersion copy() { + PkgVersion result = new PkgVersion(); + result.version = this.version; + result.files = this.files; + result.manifest = this.manifest; + result.manifestSHA512 = this.manifestSHA512; + return result; + } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java index b5b295f156f..1895b6d3069 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -20,10 +20,11 @@ package org.apache.solr.pkg; import java.lang.invoke.MethodHandles; import java.lang.ref.Reference; import java.lang.ref.SoftReference; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; + +import org.apache.solr.common.MapWriter; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.logging.MDCLoggingContext; @@ -34,7 +35,7 @@ public class PackageListeners { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String PACKAGE_VERSIONS = "PKG_VERSIONS"; - private SolrCore core; + private final SolrCore core; public PackageListeners(SolrCore core) { this.core = core; @@ -64,21 +65,23 @@ public class PackageListeners { synchronized void packagesUpdated(List pkgs) { MDCLoggingContext.setCore(core); + Listener.Ctx ctx = new Listener.Ctx(); try { for (PackageLoader.Package pkgInfo : pkgs) { - invokeListeners(pkgInfo); + invokeListeners(pkgInfo, ctx); } } finally { + ctx.runLaterTasks(core::runAsync); MDCLoggingContext.clear(); } } - private synchronized void invokeListeners(PackageLoader.Package pkg) { + private synchronized void invokeListeners(PackageLoader.Package pkg, Listener.Ctx ctx) { for (Reference ref : listeners) { Listener listener = ref.get(); if(listener == null) continue; if (listener.packageName() == null || listener.packageName().equals(pkg.name())) { - listener.changed(pkg); + listener.changed(pkg, ctx); } } } @@ -96,15 +99,41 @@ public class PackageListeners { public interface Listener { - /**Name of the package or null to loisten to all package changes - */ + /**Name of the package or null to listen to all package changes */ String packageName(); PluginInfo pluginInfo(); - void changed(PackageLoader.Package pkg); + /**A callback when the package is updated */ + void changed(PackageLoader.Package pkg, Ctx ctx); - PackageLoader.Package.Version getPackageVersion(); + default MapWriter getPackageVersion(PluginInfo.ClassName cName) { + return null; + } + class Ctx { + private Map runLater; + + /** + * If there are multiple packages to be updated and there are multiple listeners, + * This is executed after all of the {@link Listener#changed(PackageLoader.Package, Ctx)} + * calls are invoked. The name is a unique identifier that can be used by consumers to avoid duplicate + * If no deduplication is required, use null as the name + */ + public void runLater(String name, Runnable runnable) { + if (runLater == null) runLater = new LinkedHashMap<>(); + if (name == null) { + name = runnable.getClass().getSimpleName() + "@" + runnable.hashCode(); + } + runLater.put(name, runnable); + } + + private void runLaterTasks(Consumer runnableExecutor) { + if (runLater == null) return; + for (Runnable r : runLater.values()) { + runnableExecutor.accept(r); + } + } + } } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java new file mode 100644 index 00000000000..ced4bd02936 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import org.apache.lucene.analysis.util.ResourceLoaderAware; +import org.apache.solr.common.MapWriter; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.SolrClassLoader; +import org.apache.solr.core.SolrResourceLoader; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +/** + * A {@link SolrClassLoader} that is designed to listen to a set of packages. + * This class registers a listener for each package that is loaded through this. + * If any of those packages are updated, the onReload runnable is run + * */ +public class PackageListeningClassLoader implements SolrClassLoader , PackageListeners.Listener { + private final CoreContainer coreContainer; + private final SolrResourceLoader coreResourceLoader; + private final Function pkgVersionSupplier; + /** package name and the versions that we are tracking + */ + private Map packageVersions = new HashMap<>(1); + private final Runnable onReload; + + public PackageListeningClassLoader(CoreContainer coreContainer, + SolrResourceLoader coreResourceLoader, + Function pkgVersionSupplier, + Runnable onReload) { + this.coreContainer = coreContainer; + this.coreResourceLoader = coreResourceLoader; + this.pkgVersionSupplier = pkgVersionSupplier; + this.onReload = () -> { + packageVersions = new HashMap<>(); + onReload.run(); + }; + } + + + @Override + public T newInstance(String cname, Class expectedType, String... subpackages) { + PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); + if(cName.pkg == null){ + return coreResourceLoader.newInstance(cname, expectedType, subpackages); + } else { + PackageLoader.Package.Version version = findPkgVersion(cName); + return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subpackages)); + + } + } + + private PackageLoader.Package.Version findPkgVersion(PluginInfo.ClassName cName) { + PackageLoader.Package.Version theVersion = coreContainer.getPackageLoader().getPackage(cName.pkg).getLatest(pkgVersionSupplier.apply(cName.pkg)); + packageVersions.put(cName.pkg, theVersion.getPkgVersion()); + return theVersion; + } + + @Override + public MapWriter getPackageVersion(PluginInfo.ClassName cName) { + if (cName.pkg == null) return null; + PackageAPI.PkgVersion p = packageVersions.get(cName.pkg); + return p == null ? null : p::writeMap; + } + + private T applyResourceLoaderAware(PackageLoader.Package.Version version, T obj) { + if (obj instanceof ResourceLoaderAware) { + SolrResourceLoader.assertAwareCompatibility(ResourceLoaderAware.class, obj); + try { + ((ResourceLoaderAware) obj).inform(version.getLoader()); + return obj; + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + } + return obj; + } + + @Override + @SuppressWarnings({"rawtypes"}) + public T newInstance(String cname, Class expectedType, String[] subPackages, Class[] params, Object[] args) { + PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); + if (cName.pkg == null) { + return coreResourceLoader.newInstance(cname, expectedType, subPackages, params, args); + } else { + PackageLoader.Package.Version version = findPkgVersion(cName); + return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args)); + } + } + + @Override + public Class findClass(String cname, Class expectedType) { + PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); + if (cName.pkg == null) { + return coreResourceLoader.findClass(cname, expectedType); + } else { + PackageLoader.Package.Version version = findPkgVersion(cName); + return version.getLoader().findClass(cName.className, expectedType); + + } + } + + @Override + public String packageName() { + return null; + } + + @Override + public PluginInfo pluginInfo() { + return null; + } + + @Override + public void changed(PackageLoader.Package pkg, Ctx ctx) { + PackageAPI.PkgVersion currVer = packageVersions.get(pkg.name); + if (currVer == null) { + //not watching this + return; + } + String latestSupportedVersion = pkgVersionSupplier.apply(pkg.name); + if (latestSupportedVersion == null) { + //no specific version configured. use the latest + latestSupportedVersion = pkg.getLatest().getVersion(); + } + if (Objects.equals(currVer.version, latestSupportedVersion)) { + //no need to update + return; + } + ctx.runLater(null, onReload); + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java index 562f8a05881..1e2fc574adb 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java @@ -50,6 +50,8 @@ import static org.apache.lucene.util.IOUtils.closeWhileHandlingException; */ public class PackageLoader implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String LATEST = "$LATEST"; + private final CoreContainer coreContainer; private final Map packageClassLoaders = new ConcurrentHashMap<>(); @@ -289,6 +291,9 @@ public class PackageLoader implements Closeable { public String getVersion() { return version.version; } + public PackageAPI.PkgVersion getPkgVersion(){ + return version.copy(); + } @SuppressWarnings({"rawtypes"}) public Collection getFiles() { diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index 322a1d3079d..0c6fd808d71 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -19,9 +19,16 @@ package org.apache.solr.pkg; import java.io.IOException; import java.lang.invoke.MethodHandles; + import org.apache.lucene.analysis.util.ResourceLoaderAware; +import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; -import org.apache.solr.core.*; +import org.apache.solr.core.PluginBag; +import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrInfoBean; +import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,32 +62,29 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { } @Override - public void changed(PackageLoader.Package pkg) { + public void changed(PackageLoader.Package pkg, Ctx ctx) { reload(pkg); } @Override - public PackageLoader.Package.Version getPackageVersion() { - return pkgVersion; + public MapWriter getPackageVersion(PluginInfo.ClassName cName) { + return pkgVersion == null ? null : ew -> pkgVersion.writeMap(ew); } }); } - private String maxVersion() { - RequestParams.ParamSet p = core.getSolrConfig().getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS); - if (p == null) { - return null; + public static PluginBag.PluginHolder createHolder(PluginInfo info, SolrCore core, Class type, String msg) { + if(info.cName.pkg == null) { + return new PluginBag.PluginHolder(info, core.createInitInstance(info, type,msg, null)); + } else { + return new PackagePluginHolder(info, core, SolrConfig.classVsSolrPluginInfo.get(type.getName())); } - Object o = p.get().get(info.pkgName); - if (o == null || LATEST.equals(o)) return null; - return o.toString(); } - private synchronized void reload(PackageLoader.Package pkg) { - String lessThan = maxVersion(); + String lessThan = core.getSolrConfig().maxPackageVersion(info.pkgName); PackageLoader.Package.Version newest = pkg.getLatest(lessThan); if (newest == null) { log.error("No latest version available for package : {}", pkg.name()); @@ -113,7 +117,7 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { } @SuppressWarnings({"unchecked"}) - protected void initNewInstance(PackageLoader.Package.Version newest) { + protected Object initNewInstance(PackageLoader.Package.Version newest) { Object instance = SolrCore.createInstance(pluginInfo.className, pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader()); PluginBag.initInstance(instance, pluginInfo); @@ -128,6 +132,7 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { log.error("error closing plugin", e); } } + return inst; } private void handleAwareCallbacks(SolrResourceLoader loader, Object instance) { diff --git a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java index f4b0b50e353..1f183264a3e 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java @@ -34,8 +34,8 @@ import org.apache.lucene.analysis.util.TokenizerFactory; import org.apache.lucene.util.Version; import org.apache.solr.analysis.TokenizerChain; import org.apache.solr.common.SolrException; +import org.apache.solr.core.SolrClassLoader; import org.apache.solr.core.SolrConfig; -import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.util.DOMUtil; import org.apache.solr.util.plugin.AbstractPluginLoader; import org.slf4j.Logger; @@ -78,7 +78,7 @@ public final class FieldTypePluginLoader @Override - protected FieldType create( SolrResourceLoader loader, + protected FieldType create( SolrClassLoader loader, String name, String className, Node node ) throws Exception { @@ -189,7 +189,7 @@ public final class FieldTypePluginLoader // private Analyzer readAnalyzer(Node node) throws XPathExpressionException { - final SolrResourceLoader loader = schema.getResourceLoader(); + final SolrClassLoader loader = schema.getSolrClassLoader(); // parent node used to be passed in as "fieldtype" // if (!fieldtype.hasChildNodes()) return null; @@ -255,7 +255,7 @@ public final class FieldTypePluginLoader @Override @SuppressWarnings({"rawtypes"}) - protected CharFilterFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { + protected CharFilterFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception { final Map params = DOMUtil.toMap(node.getAttributes()); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, CharFilterFactory.class.getSimpleName()).toString()); @@ -306,7 +306,7 @@ public final class FieldTypePluginLoader @Override @SuppressWarnings({"rawtypes"}) - protected TokenizerFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { + protected TokenizerFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception { final Map params = DOMUtil.toMap(node.getAttributes()); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenizerFactory.class.getSimpleName()).toString()); @@ -361,7 +361,7 @@ public final class FieldTypePluginLoader { @Override @SuppressWarnings({"rawtypes"}) - protected TokenFilterFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { + protected TokenFilterFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception { final Map params = DOMUtil.toMap(node.getAttributes()); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenFilterFactory.class.getSimpleName()).toString()); diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index bd1827d836f..af2a5b1e792 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -62,6 +62,7 @@ import org.apache.solr.common.util.Cache; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.SolrClassLoader; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.core.XmlConfigFile; @@ -135,6 +136,7 @@ public class IndexSchema { protected final Version luceneVersion; protected float version; protected final SolrResourceLoader loader; + protected final SolrClassLoader solrClassLoader; protected final Properties substitutableProperties; protected Map fields = new HashMap<>(); @@ -188,6 +190,7 @@ public class IndexSchema { protected IndexSchema(Version luceneVersion, SolrResourceLoader loader, Properties substitutableProperties) { this.luceneVersion = Objects.requireNonNull(luceneVersion); this.loader = loader; + this.solrClassLoader = loader.getCore() == null? loader: loader.getCore().getSchemaPluginsLoader(); this.substitutableProperties = substitutableProperties; } @@ -208,6 +211,10 @@ public class IndexSchema { return resourceName; } + public SolrClassLoader getSolrClassLoader() { + return solrClassLoader; + } + /** Sets the name of the resource used to instantiate this schema. */ public void setResourceName(String resourceName) { this.resourceName = resourceName; @@ -391,7 +398,7 @@ public class IndexSchema { return sf.getType().getUninversionType(sf); } // else... - + // It would be nice to throw a helpful error here, with a good useful message for the user, // but unfortunately, inspite of the UninvertingReader class jdoc claims that the uninversion // process is lazy, that doesn't mean it's lazy as of "When a caller attempts ot use doc values" @@ -405,7 +412,7 @@ public class IndexSchema { } /** - * Writes the schema in schema.xml format to the given writer + * Writes the schema in schema.xml format to the given writer */ void persist(Writer writer) throws IOException { final SolrQueryResponse response = new SolrQueryResponse(); @@ -499,18 +506,18 @@ public class IndexSchema { final FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, fieldTypes, schemaAware); expression = getFieldTypeXPathExpressions(); NodeList nodes = (NodeList) xpath.evaluate(expression, document, XPathConstants.NODESET); - typeLoader.load(loader, nodes); + typeLoader.load(solrClassLoader, nodes); // load the fields Map explicitRequiredProp = loadFields(document, xpath); expression = stepsToPath(SCHEMA, SIMILARITY); // /schema/similarity Node node = (Node) xpath.evaluate(expression, document, XPathConstants.NODE); - similarityFactory = readSimilarity(loader, node); + similarityFactory = readSimilarity(solrClassLoader, node); if (similarityFactory == null) { final Class simClass = SchemaSimilarityFactory.class; // use the loader to ensure proper SolrCoreAware handling - similarityFactory = loader.newInstance(simClass.getName(), SimilarityFactory.class); + similarityFactory = solrClassLoader.newInstance(simClass.getName(), SimilarityFactory.class); similarityFactory.init(new ModifiableSolrParams()); } else { isExplicitSimilarity = true; @@ -594,12 +601,12 @@ public class IndexSchema { uniqueKeyField.required = true; requiredFields.add(uniqueKeyField); } - } + } /////////////// parse out copyField commands /////////////// // Map> cfields = new HashMap>(); // expression = "/schema/copyField"; - + dynamicCopyFields = new DynamicCopy[] {}; loadCopyFields(document, xpath); @@ -754,10 +761,10 @@ public class IndexSchema { log.error(msg); throw new SolrException(ErrorCode.SERVER_ERROR, msg); } - + registerCopyField(source, dest, maxCharsInt); } - + for (Map.Entry entry : copyFieldTargetCounts.entrySet()) { if (entry.getValue() > 1 && !entry.getKey().multiValued()) { log.warn("Field {} is not multivalued and destination for multiople {} ({})" @@ -786,7 +793,7 @@ public class IndexSchema { } return false; } - + protected boolean isValidDynamicField(List dFields, SchemaField f) { String glob = f.getName(); if (f.getDefaultValue() != null) { @@ -853,7 +860,7 @@ public class IndexSchema { * inform( SolrCore core ) function for SolrCoreAware classes. * Outside inform, this could potentially throw a ConcurrentModificationException *

- * + * * @see SolrCoreAware */ public void registerCopyField(String source, String dest, int maxChars) { @@ -863,10 +870,10 @@ public class IndexSchema { DynamicField destDynamicField = null; SchemaField destSchemaField = fields.get(dest); SchemaField sourceSchemaField = fields.get(source); - + DynamicField sourceDynamicBase = null; DynamicField destDynamicBase = null; - + boolean sourceIsDynamicFieldReference = false; boolean sourceIsExplicitFieldGlob = false; @@ -892,7 +899,7 @@ public class IndexSchema { } } } - + if (null == destSchemaField || (null == sourceSchemaField && ! sourceIsExplicitFieldGlob)) { // Go through dynamicFields array only once, collecting info for both source and dest fields, if needed for (DynamicField dynamicField : dynamicFields) { @@ -914,7 +921,7 @@ public class IndexSchema { destDynamicBase = dynamicField; } } - if (null != destSchemaField + if (null != destSchemaField && (null != sourceSchemaField || sourceIsDynamicFieldReference || sourceIsExplicitFieldGlob)) { break; } @@ -971,7 +978,7 @@ public class IndexSchema { copyFieldList.add(new CopyField(sourceSchemaField, destSchemaField, maxChars)); incrementCopyFieldTargetCount(destSchemaField); } - + private void incrementCopyFieldTargetCount(SchemaField dest) { copyFieldTargetCounts.put(dest, copyFieldTargetCounts.containsKey(dest) ? copyFieldTargetCounts.get(dest) + 1 : 1); } @@ -983,7 +990,7 @@ public class IndexSchema { dynamicCopyFields = temp; } - static SimilarityFactory readSimilarity(SolrResourceLoader loader, Node node) { + static SimilarityFactory readSimilarity(SolrClassLoader loader, Node node) { if (node==null) { return null; } else { @@ -1024,7 +1031,7 @@ public class IndexSchema { else { return new NameEquals(regex); } } - + /** Returns true if the given name matches this pattern */ abstract boolean matches(String name); @@ -1033,7 +1040,7 @@ public class IndexSchema { /** Returns the result of combining this pattern's fixed string component with the given replacement */ abstract String subst(String replacement); - + /** Returns the length of the original regex, including the asterisk, if any. */ public int length() { return regex.length(); } @@ -1076,7 +1083,7 @@ public class IndexSchema { public int compareTo(DynamicReplacement other) { return other.pattern.length() - pattern.length(); } - + /** Returns the regex used to create this instance's pattern */ public String getRegex() { return pattern.regex; @@ -1110,7 +1117,7 @@ public class IndexSchema { public static class DynamicCopy extends DynamicReplacement { private final DynamicField destination; - + private final int maxChars; public int getMaxChars() { return maxChars; } @@ -1120,7 +1127,7 @@ public class IndexSchema { final DynamicField destDynamicBase; public DynamicField getDestDynamicBase() { return destDynamicBase; } - DynamicCopy(String sourceRegex, DynamicField destination, int maxChars, + DynamicCopy(String sourceRegex, DynamicField destination, int maxChars, DynamicField sourceDynamicBase, DynamicField destDynamicBase) { super(sourceRegex); this.destination = destination; @@ -1144,7 +1151,7 @@ public class IndexSchema { return destination.makeSchemaField(targetFieldName); } - + @Override public String toString() { return destination.prototype.toString(); @@ -1165,7 +1172,7 @@ public class IndexSchema { } return null; } - + /** * Does the schema explicitly define the specified field, i.e. not as a result * of a copyField declaration? We consider it explicitly defined if it matches @@ -1198,7 +1205,7 @@ public class IndexSchema { } return false; - } + } /** * Returns the SchemaField that should be used for the specified field name, or @@ -1373,10 +1380,10 @@ public class IndexSchema { return result; } - + /** - * Check if a field is used as the destination of a copyField operation - * + * Check if a field is used as the destination of a copyField operation + * * @since solr 1.3 */ public boolean isCopyFieldTarget( SchemaField f ) { @@ -1529,14 +1536,14 @@ public class IndexSchema { /** * Returns a list of copyField directives, with optional details and optionally restricting to those * directives that contain the requested source and/or destination field names. - * + * * @param showDetails If true, source and destination dynamic bases, and explicit fields matched by source globs, * will be added to dynamic copyField directives where appropriate * @param requestedSourceFields If not null, output is restricted to those copyField directives - * with the requested source field names + * with the requested source field names * @param requestedDestinationFields If not null, output is restricted to those copyField directives - * with the requested destination field names - * @return a list of copyField directives + * with the requested destination field names + * @return a list of copyField directives */ public List> getCopyFieldProperties (boolean showDetails, Set requestedSourceFields, Set requestedDestinationFields) { @@ -1613,7 +1620,7 @@ public class IndexSchema { * Requires synchronizing on the object returned by * {@link #getSchemaUpdateLock()}. * - * @param newField the SchemaField to add + * @param newField the SchemaField to add * @param persist to persist the schema or not * @return a new IndexSchema based on this schema with newField added * @see #newField(String, String, Map) @@ -1689,7 +1696,7 @@ public class IndexSchema { } /** - * Copies this schema, deletes the named field from the copy, creates a new field + * Copies this schema, deletes the named field from the copy, creates a new field * with the same name using the given args, then rebinds any referring copy fields * to the replacement field. * @@ -1699,7 +1706,7 @@ public class IndexSchema { * Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. * * @param fieldName The name of the field to be replaced - * @param replacementFieldType The field type of the replacement field + * @param replacementFieldType The field type of the replacement field * @param replacementArgs Initialization params for the replacement field * @return a new IndexSchema based on this schema with the named field replaced */ @@ -1757,7 +1764,7 @@ public class IndexSchema { * Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. * * @param fieldNamePattern The glob for the dynamic field to be replaced - * @param replacementFieldType The field type of the replacement dynamic field + * @param replacementFieldType The field type of the replacement dynamic field * @param replacementArgs Initialization params for the replacement dynamic field * @return a new IndexSchema based on this schema with the named dynamic field replaced */ @@ -1787,10 +1794,10 @@ public class IndexSchema { /** * Copies this schema and adds the new copy fields to the copy. - * - * Requires synchronizing on the object returned by + * + * Requires synchronizing on the object returned by * {@link #getSchemaUpdateLock()} - * + * * @param source source field name * @param destinations collection of target field names * @param maxChars max number of characters to copy from the source to each @@ -1812,7 +1819,7 @@ public class IndexSchema { * Requires synchronizing on the object returned by * {@link #getSchemaUpdateLock()}. * - * @param copyFields Key is the name of the source field name, value is a collection of target field names. + * @param copyFields Key is the name of the source field name, value is a collection of target field names. * Each corresponding copy field directives must exist. * @return The new Schema with the copy fields deleted */ @@ -1824,8 +1831,8 @@ public class IndexSchema { /** - * Returns a SchemaField if the given fieldName does not already - * exist in this schema, and does not match any dynamic fields + * Returns a SchemaField if the given fieldName does not already + * exist in this schema, and does not match any dynamic fields * in this schema. The resulting SchemaField can be used in a call * to {@link #addField(SchemaField)}. * @@ -1842,8 +1849,8 @@ public class IndexSchema { } /** - * Returns a SchemaField if the given dynamic field glob does not already - * exist in this schema, and does not match any dynamic fields + * Returns a SchemaField if the given dynamic field glob does not already + * exist in this schema, and does not match any dynamic fields * in this schema. The resulting SchemaField can be used in a call * to {@link #addField(SchemaField)}. * @@ -1904,15 +1911,15 @@ public class IndexSchema { } /** - * Copies this schema, deletes the named field type from the copy, creates a new field type + * Copies this schema, deletes the named field type from the copy, creates a new field type * with the same name using the given args, rebuilds fields and dynamic fields of the given * type, then rebinds any referring copy fields to the rebuilt fields. - * + * *

* The schema will not be persisted. *

* Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. - * + * * @param typeName The name of the field type to be replaced * @param replacementClassName The class name of the replacement field type * @param replacementArgs Initialization params for the replacement field type @@ -1944,14 +1951,14 @@ public class IndexSchema { protected String getFieldTypeXPathExpressions() { // /schema/fieldtype | /schema/fieldType | /schema/types/fieldtype | /schema/types/fieldType String expression = stepsToPath(SCHEMA, FIELD_TYPE.toLowerCase(Locale.ROOT)) // backcompat(?) - + XPATH_OR + stepsToPath(SCHEMA, FIELD_TYPE) - + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE.toLowerCase(Locale.ROOT)) - + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE); + + XPATH_OR + stepsToPath(SCHEMA, FIELD_TYPE) + + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE.toLowerCase(Locale.ROOT)) + + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE); return expression; } /** - * Helper method that returns true if the {@link #ROOT_FIELD_NAME} uses the exact + * Helper method that returns true if the {@link #ROOT_FIELD_NAME} uses the exact * same 'type' as the {@link #getUniqueKeyField()} * * @lucene.internal diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java index 7f114d937a9..d44e9619cc2 100644 --- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java @@ -1301,7 +1301,7 @@ public final class ManagedIndexSchema extends IndexSchema { Map newFieldTypes = new HashMap<>(); List schemaAwareList = new ArrayList<>(); FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, newFieldTypes, schemaAwareList); - typeLoader.loadSingle(loader, FieldTypeXmlAdapter.toNode(options)); + typeLoader.loadSingle(solrClassLoader, FieldTypeXmlAdapter.toNode(options)); FieldType ft = newFieldTypes.get(typeName); if (!schemaAwareList.isEmpty()) schemaAware.addAll(schemaAwareList); diff --git a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java index 3ba172030c6..84265e2d97a 100644 --- a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java +++ b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java @@ -79,7 +79,7 @@ public class PreAnalyzedField extends TextField implements HasImplicitIndexAnaly parser = new SimplePreAnalyzedParser(); } else { try { - Class implClazz = schema.getResourceLoader().findClass(implName, PreAnalyzedParser.class); + Class implClazz = schema.getSolrClassLoader().findClass(implName, PreAnalyzedParser.class); Constructor c = implClazz.getConstructor(new Class[0]); parser = (PreAnalyzedParser) c.newInstance(new Object[0]); } catch (Exception e) { diff --git a/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java b/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java index c98f80600db..568338ac209 100644 --- a/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java +++ b/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java @@ -23,7 +23,7 @@ import java.util.Objects; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.core.SolrClassLoader; import org.apache.solr.util.DOMUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,7 +86,7 @@ public abstract class AbstractPluginLoader * @param node - the XML node defining this plugin */ @SuppressWarnings("unchecked") - protected T create( SolrResourceLoader loader, String name, String className, Node node ) throws Exception + protected T create(SolrClassLoader loader, String name, String className, Node node ) throws Exception { return loader.newInstance(className, pluginClassType, getDefaultPackages()); } @@ -135,7 +135,7 @@ public abstract class AbstractPluginLoader * If a default element is defined, it will be returned from this function. * */ - public T load( SolrResourceLoader loader, NodeList nodes ) + public T load(SolrClassLoader loader, NodeList nodes ) { List info = new ArrayList<>(); T defaultPlugin = null; @@ -225,7 +225,7 @@ public abstract class AbstractPluginLoader * The created class for the plugin will be returned from this function. * */ - public T loadSingle(SolrResourceLoader loader, Node node) { + public T loadSingle(SolrClassLoader loader, Node node) { List info = new ArrayList<>(); T plugin = null; diff --git a/solr/core/src/test-files/runtimecode/schema-plugins.jar.bin b/solr/core/src/test-files/runtimecode/schema-plugins.jar.bin new file mode 100644 index 00000000000..4effbbafeb6 Binary files /dev/null and b/solr/core/src/test-files/runtimecode/schema-plugins.jar.bin differ diff --git a/solr/core/src/test-files/runtimecode/sig.txt b/solr/core/src/test-files/runtimecode/sig.txt index 59beb47ca6e..2a42c7082c5 100644 --- a/solr/core/src/test-files/runtimecode/sig.txt +++ b/solr/core/src/test-files/runtimecode/sig.txt @@ -112,6 +112,11 @@ openssl dgst -sha512 expressible.jar.bin 3474a1414c8329c71ef5db2d3eb6e870363bdd7224a836aab561dccf5e8bcee4974ac799e72398c7e0b0c01972bab1c7454c8a4e791a8865bb676c0440627388 +openssl dgst -sha512 schema-plugins.jar.bin + +9299d8d5f8b358b66f113a0a91a0f55db09a41419872cb56e6e4ee402645487b75b17ec515f8035352e7fdcfac9e2e861f371d1e601869859384807b19abc2fb + + =============sha256============================ openssl dgst -sha256 runtimelibs.jar.bin diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java index 18a5f34d3ac..3ec43c9a086 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -210,7 +210,7 @@ public class TestCodecSupport extends SolrTestCaseJ4 { try { CoreDescriptor cd = new CoreDescriptor(newCoreName, testSolrHome.resolve(newCoreName), coreContainer); c = new SolrCore(coreContainer, cd, - new ConfigSet("fakeConfigset", config, schema, null, true)); + new ConfigSet("fakeConfigset", config, forceFetch -> schema, null, true)); assertNull(coreContainer.registerCore(cd, c, false, false)); h.coreName = newCoreName; assertEquals("We are not using the correct core", "solrconfig_codec2.xml", h.getCore().getConfigResource()); diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index d20efca7dcf..a54a3b09fe1 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -25,10 +25,7 @@ import java.util.concurrent.Callable; import org.apache.commons.codec.digest.DigestUtils; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoaderAware; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.*; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; @@ -39,6 +36,7 @@ import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.request.beans.Package; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; @@ -46,6 +44,7 @@ import org.apache.solr.common.MapWriterMap; import org.apache.solr.common.NavigableObject; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; @@ -642,6 +641,139 @@ public class TestPackages extends SolrCloudTestCase { } } + @SuppressWarnings("rawtypes") + public void testSchemaPlugins() throws Exception { + String COLLECTION_NAME = "testSchemaLoadingColl"; + System.setProperty("managed.schema.mutable", "true"); + + MiniSolrCloudCluster cluster = + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-managed")) + .configure(); + try { + String FILE1 = "/schemapkg/schema-plugins.jar"; + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster); + postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1, + "iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A=="); + + Package.AddVersion add = new Package.AddVersion(); + add.version = "1.0"; + add.pkg = "schemapkg"; + add.files = Arrays.asList(new String[]{FILE1}); + V2Request req = new V2Request.Builder("/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + req.process(cluster.getSolrClient()); + + TestDistribPackageStore.assertResponseValues(10, + () -> new V2Request.Builder("/cluster/package"). + withMethod(SolrRequest.METHOD.GET) + .build().process(cluster.getSolrClient()), + Utils.makeMap( + ":result:packages:schemapkg[0]:version", "1.0", + ":result:packages:schemapkg[0]:files[0]", FILE1 + )); + + CollectionAdminRequest + .createCollection(COLLECTION_NAME, "conf", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + String addFieldTypeAnalyzerWithClass = "{\n" + + "'add-field-type' : {" + + " 'name' : 'myNewTextFieldWithAnalyzerClass',\n" + + " 'class':'schemapkg:my.pkg.MyTextField',\n" + + " 'analyzer' : {\n" + + " 'luceneMatchVersion':'5.0.0'" ; +// + ",\n" + +// " 'class':'schemapkg:my.pkg.MyWhitespaceAnalyzer'\n"; + String charFilters = + " 'charFilters' : [{\n" + + " 'class':'schemapkg:my.pkg.MyPatternReplaceCharFilterFactory',\n" + + " 'replacement':'$1$1',\n" + + " 'pattern':'([a-zA-Z])\\\\\\\\1+'\n" + + " }],\n"; + String tokenizer = + " 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n"; + String filters = + " 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n"; + String suffix = " }\n" + + "}}"; + cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") { + + @Override + public RequestWriter.ContentWriter getContentWriter(String expectedType) { + return new RequestWriter.StringPayloadContentWriter(addFieldTypeAnalyzerWithClass + ',' + charFilters + tokenizer + filters + suffix, CommonParams.JSON_MIME); + } + + @Override + public SolrParams getParams() { + return null; + } + + @Override + public String getCollection() { + return COLLECTION_NAME; + } + + @Override + public SolrResponse createResponse(SolrClient client) { + return new SolrResponseBase(); + } + }); + verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass", + Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"1.0", + ":fieldType:analyzer:tokenizer:_packageinfo_:version","1.0", + ":fieldType:_packageinfo_:version","1.0")); + + add = new Package.AddVersion(); + add.version = "2.0"; + add.pkg = "schemapkg"; + add.files = Arrays.asList(new String[]{FILE1}); + req = new V2Request.Builder("/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + req.process(cluster.getSolrClient()); + + TestDistribPackageStore.assertResponseValues(10, + () -> new V2Request.Builder("/cluster/package"). + withMethod(SolrRequest.METHOD.GET) + .build().process(cluster.getSolrClient()), + Utils.makeMap( + ":result:packages:schemapkg[0]:version", "2.0", + ":result:packages:schemapkg[0]:files[0]", FILE1 + )); + + verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass", + Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"2.0", + ":fieldType:analyzer:tokenizer:_packageinfo_:version","2.0", + ":fieldType:_packageinfo_:version","2.0")); + + } finally { + cluster.shutdown(); + } + + } + @SuppressWarnings({"rawtypes","unchecked"}) + private void verifySchemaComponent(SolrClient client, String COLLECTION_NAME, String path, + Map expected) throws Exception { + SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, + WT, JAVABIN, + "meta", "true")); + + GenericSolrRequest req = new GenericSolrRequest(SolrRequest.METHOD.GET,path + , params); + TestDistribPackageStore.assertResponseValues(10, + client, + req, expected); + } + public static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { ByteBuffer fileContent = getFileContent(fname); String sha512 = DigestUtils.sha512Hex(fileContent.array());