From 9466af576a4a9d3cd750438123063928329fbb46 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Wed, 13 Jan 2021 22:28:01 +1100 Subject: [PATCH] SOLR-14155: Load all other SolrCore plugins from packages (#1666) --- solr/CHANGES.txt | 2 + .../apache/solr/core/DirectoryFactory.java | 2 +- .../java/org/apache/solr/core/PluginBag.java | 1 - .../java/org/apache/solr/core/SolrCore.java | 18 ++-- .../apache/solr/core/SolrResourceLoader.java | 70 ++++++++++++-- .../solr/handler/SolrConfigHandler.java | 29 +++--- .../solr/handler/component/SearchHandler.java | 26 ++--- .../org/apache/solr/pkg/PackageListeners.java | 15 ++- .../solr/pkg/PackageListeningClassLoader.java | 73 +++++++++----- .../apache/solr/pkg/PackagePluginHolder.java | 16 +++- .../org/apache/solr/search/CacheConfig.java | 44 ++++++--- .../org/apache/solr/update/UpdateHandler.java | 13 +-- .../solr/configsets/conf2/conf/schema.xml | 43 +++++++++ .../solr/configsets/conf2/conf/solrconfig.xml | 68 +++++++++++++ .../org/apache/solr/pkg/TestPackages.java | 95 +++++++++++++++---- 15 files changed, 403 insertions(+), 112 deletions(-) create mode 100644 solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml create mode 100644 solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a62ca52eeee..7c8a26ff12d 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -249,6 +249,8 @@ Improvements * SOLR-15040: Improvements to postlogs timestamp handling (Joel Bernstein) +* SOLR-14155: Load all other SolrCore plugins from packages (noble) + Optimizations --------------------- * SOLR-14975: Optimize CoreContainer.getAllCoreNames, getLoadedCoreNames and getCoreDescriptors. (Bruno Roustant) diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java index 57366921c05..619e7c9771c 100644 --- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java @@ -419,7 +419,7 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin, final DirectoryFactory dirFactory; if (info != null) { log.debug(info.className); - dirFactory = config.getResourceLoader().newInstance(info.className, DirectoryFactory.class); + dirFactory = config.getResourceLoader().newInstance (info, DirectoryFactory.class, true); // allow DirectoryFactory instances to access the CoreContainer dirFactory.initCoreContainer(cc); dirFactory.init(info.initArgs); diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index bdafa722690..427cd7eb13f 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -396,7 +396,6 @@ public class PluginBag implements AutoCloseable { if (pluginInfo != null) return pluginInfo.className; return null; } - public PluginInfo getPluginInfo() { return pluginInfo; } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index a388bf3a3d9..c01f4c4b5c2 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -755,7 +755,7 @@ public final class SolrCore implements SolrInfoBean, Closeable { IndexReaderFactory indexReaderFactory; PluginInfo info = solrConfig.getPluginInfo(IndexReaderFactory.class.getName()); if (info != null) { - indexReaderFactory = resourceLoader.newInstance(info.className, IndexReaderFactory.class); + indexReaderFactory = resourceLoader.newInstance(info, IndexReaderFactory.class, true); indexReaderFactory.init(info.initArgs); } else { indexReaderFactory = new StandardIndexReaderFactory(); @@ -955,6 +955,7 @@ public final class SolrCore implements SolrInfoBean, Closeable { this.solrConfig = configSet.getSolrConfig(); this.resourceLoader = configSet.getSolrConfig().getResourceLoader(); + this.resourceLoader.initCore(this); IndexSchema schema = configSet.getIndexSchema(); this.configSetProperties = configSet.getProperties(); @@ -1241,7 +1242,7 @@ public final class SolrCore implements SolrInfoBean, Closeable { private void checkVersionFieldExistsInSchema(IndexSchema schema, CoreDescriptor coreDescriptor) { if (null != coreDescriptor.getCloudDescriptor()) { - // we are evidently running in cloud mode. + // we are evidently running in cloud mode. // // In cloud mode, version field is required for correct consistency // ideally this check would be more fine grained, and individual features @@ -1405,7 +1406,7 @@ public final class SolrCore implements SolrInfoBean, Closeable { final PluginInfo info = solrConfig.getPluginInfo(CodecFactory.class.getName()); final CodecFactory factory; if (info != null) { - factory = resourceLoader.newInstance(info.className, CodecFactory.class); + factory = resourceLoader.newInstance( info, CodecFactory.class, true); factory.init(info.initArgs); } else { factory = new CodecFactory() { @@ -1443,8 +1444,8 @@ public final class SolrCore implements SolrInfoBean, Closeable { final StatsCache cache; PluginInfo pluginInfo = solrConfig.getPluginInfo(StatsCache.class.getName()); if (pluginInfo != null && pluginInfo.className != null && pluginInfo.className.length() > 0) { - cache = createInitInstance(pluginInfo, StatsCache.class, null, - LocalStatsCache.class.getName()); + cache = resourceLoader.newInstance( pluginInfo, StatsCache.class, true); + initPlugin(pluginInfo ,cache); if (log.isDebugEnabled()) { log.debug("Using statsCache impl: {}", cache.getClass().getName()); } @@ -2134,7 +2135,7 @@ public final class SolrCore implements SolrInfoBean, Closeable { newReader = currentReader; } - // for now, turn off caches if this is for a realtime reader + // for now, turn off caches if this is for a realtime reader // (caches take a little while to instantiate) final boolean useCaches = !realtime; final String newName = realtime ? "realtime" : "main"; @@ -3177,7 +3178,7 @@ public final class SolrCore implements SolrInfoBean, Closeable { public void cleanupOldIndexDirectories(boolean reload) { final DirectoryFactory myDirFactory = getDirectoryFactory(); final String myDataDir = getDataDir(); - final String myIndexDir = getNewIndexDir(); // ensure the latest replicated index is protected + final String myIndexDir = getNewIndexDir(); // ensure the latest replicated index is protected final String coreName = getName(); if (myDirFactory != null && myDataDir != null && myIndexDir != null) { Thread cleanupThread = new Thread(() -> { @@ -3280,6 +3281,9 @@ public final class SolrCore implements SolrInfoBean, Closeable { this.coreName = coreName; this.coreId = coreId; } + public void reload() { + coreContainer.reload(coreName, coreId); + } public void withCore(Consumer r) { try(SolrCore core = coreContainer.getCore(coreName, coreId)) { diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index fa183c62b72..288259ac6e2 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -64,6 +65,7 @@ import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.handler.component.ShardHandlerFactory; import org.apache.solr.logging.DeprecationLog; import org.apache.solr.pkg.PackageListeningClassLoader; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.rest.RestManager; @@ -100,6 +102,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL private CoreContainer coreContainer; private PackageListeningClassLoader schemaLoader ; + private PackageListeningClassLoader coreReloadingClassLoader ; private final List waitingForCore = Collections.synchronizedList(new ArrayList()); private final List infoMBeans = Collections.synchronizedList(new ArrayList()); private final List waitingForResources = Collections.synchronizedList(new ArrayList()); @@ -109,7 +112,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL // Provide a registry so that managed resources can register themselves while the XML configuration // documents are being parsed ... after all are registered, they are asked by the RestManager to // initialize themselves. This two-step process is required because not all resources are available - // (such as the SolrZkClient) when XML docs are being parsed. + // (such as the SolrZkClient) when XML docs are being parsed. private RestManager.Registry managedResourceRegistry; /** @see #reloadLuceneSPI() */ @@ -165,7 +168,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL if (instanceDir == null) { throw new NullPointerException("SolrResourceLoader instanceDir must be non-null"); } - + this.instanceDir = instanceDir; log.debug("new SolrResourceLoader for directory: '{}'", this.instanceDir); @@ -645,15 +648,29 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL } } + void initCore(SolrCore core) { + this.coreName = core.getName(); + this.config = core.getSolrConfig(); + this.coreId = core.uniqueId; + this.coreContainer = core.getCoreContainer(); + SolrCore.Provider coreProvider = core.coreProvider; + + this.coreReloadingClassLoader = new PackageListeningClassLoader(core.getCoreContainer(), + this, s -> config.maxPackageVersion(s), null){ + @Override + protected void doReloadAction(Ctx ctx) { + log.info("Core reloading classloader issued reload for: {}/{} ", coreName, coreId); + coreProvider.reload(); + } + }; + core.getPackageListeners().addListener(coreReloadingClassLoader, true); + + } /** * Tell all {@link SolrCoreAware} instances about the SolrCore */ public void inform(SolrCore core) { - this.coreName = core.getName(); - this.config = core.getSolrConfig(); - this.coreId = core.uniqueId; - this.coreContainer = core.getCoreContainer(); if(getSchemaLoader() != null) core.getPackageListeners().addListener(schemaLoader); // make a copy to avoid potential deadlock of a callback calling newInstance and trying to @@ -834,6 +851,47 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL public List getInfoMBeans() { return Collections.unmodifiableList(infoMBeans); } + /** + * Load a class using an appropriate {@link SolrResourceLoader} depending of the package on that class + * @param registerCoreReloadListener register a listener for the package and reload the core if the package is changed. + * Use this sparingly. This will result in core reloads across all the cores in + * all collections using this configset + */ + public Class findClass( PluginInfo info, Class type, boolean registerCoreReloadListener) { + if(info.cName.pkg == null) return findClass(info.className, type); + return _classLookup(info, + (Function>) ver -> ver.getLoader().findClass(info.cName.className, type), registerCoreReloadListener); + + } + + + private T _classLookup(PluginInfo info, Function fun, boolean registerCoreReloadListener ) { + PluginInfo.ClassName cName = info.cName; + if (registerCoreReloadListener) { + if (coreReloadingClassLoader == null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Core not set"); + } + return fun.apply(coreReloadingClassLoader.findPackageVersion(cName, true)); + } else { + return fun.apply(coreReloadingClassLoader.findPackageVersion(cName, false)); + } + } + + /** + *Create a n instance of a class using an appropriate {@link SolrResourceLoader} depending on the package of that class + * @param registerCoreReloadListener register a listener for the package and reload the core if the package is changed. + * Use this sparingly. This will result in core reloads across all the cores in + * all collections using this configset + */ + public T newInstance(PluginInfo info, Class type, boolean registerCoreReloadListener) { + if(info.cName.pkg == null) { + return newInstance(info.cName.className == null? + type.getName(): + info.cName.className , + type); + } + return _classLookup( info, version -> version.getLoader().newInstance(info.cName.className, type), registerCoreReloadListener); + } private PackageListeningClassLoader createSchemaLoader() { CoreContainer cc = getCoreContainer(); diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index 0b97896f504..db8ec17bde5 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -48,6 +48,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.io.stream.expr.Expressible; import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.ZkSolrResourceLoader; +import org.apache.solr.common.MapSerializable; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; @@ -69,6 +70,7 @@ import org.apache.solr.core.RequestParams; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; @@ -250,25 +252,28 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa String componentName = req.getParams().get("componentName"); if (componentName != null) { @SuppressWarnings({"rawtypes"}) - Map map = (Map) val.get(parts.get(1)); - if (map != null) { - Object o = map.get(componentName); - val.put(parts.get(1), makeMap(componentName, o)); + Map pluginNameVsPluginInfo = (Map) val.get(parts.get(1)); + if (pluginNameVsPluginInfo != null) { + @SuppressWarnings({"rawtypes"}) + Object o = pluginNameVsPluginInfo instanceof MapSerializable ? + pluginNameVsPluginInfo: + pluginNameVsPluginInfo.get(componentName); + @SuppressWarnings({"rawtypes"}) + Map pluginInfo = o instanceof MapSerializable? ((MapSerializable) o).toMap(new LinkedHashMap<>()): (Map) o; + val.put(parts.get(1),pluginNameVsPluginInfo instanceof PluginInfo? pluginInfo : makeMap(componentName, pluginInfo)); if (req.getParams().getBool("meta", false)) { // meta=true is asking for the package info of the plugin // We go through all the listeners and see if there is one registered for this plugin List listeners = req.getCore().getPackageListeners().getListeners(); for (PackageListeners.Listener listener : listeners) { - PluginInfo info = listener.pluginInfo(); - if(info == null) continue; - if (info.type.equals(parts.get(1)) && info.name.equals(componentName)) { - if (o instanceof Map) { - @SuppressWarnings({"rawtypes"}) - Map m1 = (Map) o; - m1.put("_packageinfo_", listener.getPackageVersion(info.cName)); + Map infos = listener.packageDetails(); + if(infos == null || infos.isEmpty()) continue; + infos.forEach((s, mapWriter) -> { + if(s.equals(pluginInfo.get("class"))) { + (pluginInfo).put("_packageinfo_", mapWriter); } - } + }); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index f5c2474c4a8..c0c536fbca7 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -16,16 +16,6 @@ */ package org.apache.solr.handler.component; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.commons.lang3.StringUtils; import org.apache.lucene.index.ExitableDirectoryReader; import org.apache.lucene.search.TotalHits; @@ -44,6 +34,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrQueryRequest; @@ -61,10 +52,13 @@ import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.common.params.CommonParams.DISTRIB; -import static org.apache.solr.common.params.CommonParams.FAILURE; -import static org.apache.solr.common.params.CommonParams.PATH; -import static org.apache.solr.common.params.CommonParams.STATUS; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.invoke.MethodHandles; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.solr.common.params.CommonParams.*; /** @@ -162,8 +156,8 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, } @Override - public PluginInfo pluginInfo() { - return null; + public Map packageDetails() { + return Collections.emptyMap(); } @Override diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java index 1895b6d3069..1281d999236 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -47,6 +47,14 @@ public class PackageListeners { public synchronized void addListener(Listener listener) { listeners.add(new SoftReference<>(listener)); + } + + public synchronized void addListener(Listener listener, boolean addFirst) { + if(addFirst) { + listeners.add(0, new SoftReference<>(listener)); + } else { + addListener(listener); + } } @@ -71,7 +79,7 @@ public class PackageListeners { invokeListeners(pkgInfo, ctx); } } finally { - ctx.runLaterTasks(core::runAsync); + ctx.runLaterTasks(r -> core.getCoreContainer().runAsync(r)); MDCLoggingContext.clear(); } } @@ -102,7 +110,10 @@ public class PackageListeners { /**Name of the package or null to listen to all package changes */ String packageName(); - PluginInfo pluginInfo(); + /** fetch the package versions of class names + * + */ + Map packageDetails(); /**A callback when the package is updated */ void changed(PackageLoader.Package pkg, Ctx ctx); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java index bbdf66b6cc2..14617aa9e61 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -26,9 +26,8 @@ import org.apache.solr.common.cloud.SolrClassLoader; import org.apache.solr.core.SolrResourceLoader; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; /** * A {@link SolrClassLoader} that is designed to listen to a set of packages. @@ -37,22 +36,29 @@ import java.util.function.Function; * */ public class PackageListeningClassLoader implements SolrClassLoader , PackageListeners.Listener { private final CoreContainer coreContainer; - private final SolrResourceLoader coreResourceLoader; + private final SolrClassLoader fallbackClassLoader; private final Function pkgVersionSupplier; /** package name and the versions that we are tracking */ - private Map packageVersions = new HashMap<>(1); + private Map packageVersions = new ConcurrentHashMap<>(1); + private Map classNameVsPackageName = new ConcurrentHashMap<>(); private final Runnable onReload; + /** + * @param fallbackClassLoader The {@link SolrClassLoader} to use if no package is specified + * @param pkgVersionSupplier Get the version configured for a given package + * @param onReload The callback function that should be run if a package is updated + */ public PackageListeningClassLoader(CoreContainer coreContainer, - SolrResourceLoader coreResourceLoader, + SolrClassLoader fallbackClassLoader, Function pkgVersionSupplier, Runnable onReload) { this.coreContainer = coreContainer; - this.coreResourceLoader = coreResourceLoader; + this.fallbackClassLoader = fallbackClassLoader; this.pkgVersionSupplier = pkgVersionSupplier; this.onReload = () -> { - packageVersions = new HashMap<>(); + packageVersions = new ConcurrentHashMap<>(); + classNameVsPackageName = new ConcurrentHashMap<>(); onReload.run(); }; } @@ -61,18 +67,28 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis @Override public T newInstance(String cname, Class expectedType, String... subpackages) { PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); - if(cName.pkg == null){ - return coreResourceLoader.newInstance(cname, expectedType, subpackages); + if(cName.pkg == null) { + return fallbackClassLoader.newInstance(cname, expectedType, subpackages); } else { - PackageLoader.Package.Version version = findPkgVersion(cName); - return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subpackages)); - + PackageLoader.Package.Version version = findPackageVersion(cName, true); + T obj = version.getLoader().newInstance(cName.className, expectedType, subpackages); + classNameVsPackageName.put(cName.original, cName.pkg); + return applyResourceLoaderAware(version, obj); } } - private PackageLoader.Package.Version findPkgVersion(PluginInfo.ClassName cName) { + + /** + * This looks up for package and also listens for that package if required + * @param cName The class name + */ + public PackageLoader.Package.Version findPackageVersion(PluginInfo.ClassName cName, boolean registerListener) { PackageLoader.Package.Version theVersion = coreContainer.getPackageLoader().getPackage(cName.pkg).getLatest(pkgVersionSupplier.apply(cName.pkg)); - packageVersions.put(cName.pkg, theVersion.getPkgVersion()); + if(registerListener) { + classNameVsPackageName.put(cName.original, cName.pkg); + PackageAPI.PkgVersion pkgVersion = theVersion.getPkgVersion(); + if(pkgVersion !=null) packageVersions.put(cName.pkg, pkgVersion); + } return theVersion; } @@ -101,10 +117,12 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis public T newInstance(String cname, Class expectedType, String[] subPackages, Class[] params, Object[] args) { PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); if (cName.pkg == null) { - return coreResourceLoader.newInstance(cname, expectedType, subPackages, params, args); + return fallbackClassLoader.newInstance(cname, expectedType, subPackages, params, args); } else { - PackageLoader.Package.Version version = findPkgVersion(cName); - return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args)); + PackageLoader.Package.Version version = findPackageVersion(cName, true); + T obj = version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args); + classNameVsPackageName.put(cName.original, cName.pkg); + return applyResourceLoaderAware(version, obj); } } @@ -112,10 +130,12 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis public Class findClass(String cname, Class expectedType) { PluginInfo.ClassName cName = new PluginInfo.ClassName(cname); if (cName.pkg == null) { - return coreResourceLoader.findClass(cname, expectedType); + return fallbackClassLoader.findClass(cname, expectedType); } else { - PackageLoader.Package.Version version = findPkgVersion(cName); - return version.getLoader().findClass(cName.className, expectedType); + PackageLoader.Package.Version version = findPackageVersion(cName, true); + Class klas = version.getLoader().findClass(cName.className, expectedType); + classNameVsPackageName.put(cName.original, cName.pkg); + return klas; } } @@ -126,8 +146,10 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis } @Override - public PluginInfo pluginInfo() { - return null; + public Map packageDetails() { + Map result = new LinkedHashMap<>(); + classNameVsPackageName.forEach((k, v) -> result.put(k, packageVersions.get(v))); + return result; } @Override @@ -146,6 +168,11 @@ public class PackageListeningClassLoader implements SolrClassLoader , PackageLis //no need to update return; } + doReloadAction(ctx); + } + + protected void doReloadAction(Ctx ctx) { + if(onReload == null) return; ctx.runLater(null, onReload); } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index 332de74f962..79626de61dc 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -19,6 +19,8 @@ package org.apache.solr.pkg; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.Map; import org.apache.lucene.util.ResourceLoaderAware; import org.apache.solr.common.MapWriter; @@ -57,8 +59,8 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { } @Override - public PluginInfo pluginInfo() { - return info; + public Map packageDetails() { + return Collections.singletonMap(info.cName.original, pkgVersion.getPkgVersion()); } @Override @@ -75,6 +77,12 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { }); } + public static PluginBag.PluginHolder createHolder(T inst, Class type) { + SolrConfig.SolrPluginInfo plugin = SolrConfig.classVsSolrPluginInfo.get(type.getName()); + PluginInfo info = new PluginInfo(plugin.tag, Collections.singletonMap("class", inst.getClass().getName())); + return new PluginBag.PluginHolder(info,inst); + } + public static PluginBag.PluginHolder createHolder(PluginInfo info, SolrCore core, Class type, String msg) { if(info.cName.pkg == null) { return new PluginBag.PluginHolder(info, core.createInitInstance(info, type,msg, null)); @@ -108,7 +116,7 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { if (log.isInfoEnabled()) { log.info("loading plugin: {} -> {} using package {}:{}", - pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion()); + pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion()); } initNewInstance(newest, core); @@ -119,7 +127,7 @@ public class PackagePluginHolder extends PluginBag.PluginHolder { @SuppressWarnings({"unchecked"}) protected Object initNewInstance(PackageLoader.Package.Version newest, SolrCore core) { Object instance = SolrCore.createInstance(pluginInfo.className, - pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader()); + pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader()); PluginBag.initInstance(instance, pluginInfo); handleAwareCallbacks(newest.getLoader(), instance, core); T old = inst; diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java index d8c41c6bd84..1520f80620f 100644 --- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java +++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java @@ -23,11 +23,15 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import org.apache.solr.common.MapSerializable; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.DOMUtil; import org.apache.solr.common.util.StrUtils; + +import org.apache.solr.core.PluginInfo; + import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; import org.slf4j.Logger; @@ -46,11 +50,17 @@ import static org.apache.solr.common.params.CommonParams.NAME; */ public class CacheConfig implements MapSerializable{ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - + private String nodeName; + + /** + * When this object is created, the core is not yet available . So, if the class is to be + * loaded from a package we should have a corresponding core + * + */ @SuppressWarnings({"rawtypes"}) - private Class clazz; + private Supplier> clazz; private Map args; private CacheRegenerator regenerator; @@ -64,9 +74,10 @@ public class CacheConfig implements MapSerializable{ @SuppressWarnings({"rawtypes"}) public CacheConfig(Class clazz, Map args, CacheRegenerator regenerator) { - this.clazz = clazz; + this.clazz = () -> clazz; this.args = args; this.regenerator = regenerator; + this.nodeName = args.get(NAME); } public CacheRegenerator getRegenerator() { @@ -85,7 +96,7 @@ public class CacheConfig implements MapSerializable{ Node node = nodes.item(i); if ("true".equals(DOMUtil.getAttrOrDefault(node, "enabled", "true"))) { CacheConfig config = getConfig(solrConfig, node.getNodeName(), - DOMUtil.toMap(node.getAttributes()), configPath); + DOMUtil.toMap(node.getAttributes()), configPath); result.put(config.args.get(NAME), config); } } @@ -133,20 +144,32 @@ public class CacheConfig implements MapSerializable{ SolrResourceLoader loader = solrConfig.getResourceLoader(); config.cacheImpl = config.args.get("class"); - if(config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache"; + if (config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache"; + config.clazz = new Supplier<>() { + @SuppressWarnings("rawtypes") + Class loadedClass; + + @Override + @SuppressWarnings("rawtypes") + public Class get() { + if (loadedClass != null) return loadedClass; + return loadedClass = (Class) loader.findClass( + new PluginInfo("cache", Collections.singletonMap("class", config.cacheImpl)), + SolrCache.class, true); + } + }; config.regenImpl = config.args.get("regenerator"); - config.clazz = loader.findClass(config.cacheImpl, SolrCache.class); if (config.regenImpl != null) { config.regenerator = loader.newInstance(config.regenImpl, CacheRegenerator.class); } - + return config; } @SuppressWarnings({"rawtypes"}) public SolrCache newInstance() { try { - SolrCache cache = clazz.getConstructor().newInstance(); + SolrCache cache = clazz.get().getConstructor().newInstance(); persistence[0] = cache.init(args, persistence[0], regenerator); return cache; } catch (Exception e) { @@ -158,11 +181,8 @@ public class CacheConfig implements MapSerializable{ } @Override - @SuppressWarnings({"unchecked"}) public Map toMap(Map map) { - @SuppressWarnings({"rawtypes"}) - Map result = Collections.unmodifiableMap(args); - return result; + return new HashMap<>(args); } public String getNodeName() { diff --git a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java index 14b78af267a..ade92401370 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java @@ -20,12 +20,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.Vector; -import org.apache.solr.core.DirectoryFactory; -import org.apache.solr.core.HdfsDirectoryFactory; -import org.apache.solr.core.PluginInfo; -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrEventListener; -import org.apache.solr.core.SolrInfoBean; +import org.apache.solr.core.*; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.SchemaField; @@ -108,7 +103,7 @@ public abstract class UpdateHandler implements SolrInfoBean { public UpdateHandler(SolrCore core) { this(core, null); } - + public UpdateHandler(SolrCore core, UpdateLog updateLog) { this.core=core; idField = core.getLatestSchema().getUniqueKeyField(); @@ -123,8 +118,8 @@ public abstract class UpdateHandler implements SolrInfoBean { if (dirFactory instanceof HdfsDirectoryFactory) { ulog = new HdfsUpdateLog(((HdfsDirectoryFactory)dirFactory).getConfDir()); } else { - String className = ulogPluginInfo.className == null ? UpdateLog.class.getName() : ulogPluginInfo.className; - ulog = core.getResourceLoader().newInstance(className, UpdateLog.class); + ulog = ulogPluginInfo.className == null ? new UpdateLog(): + core.getResourceLoader().newInstance(ulogPluginInfo, UpdateLog.class, true); } if (!core.isReloaded() && !dirFactory.isPersistent()) { diff --git a/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml b/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml new file mode 100644 index 00000000000..85f7ed35453 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/conf2/conf/schema.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + id + + + + + + diff --git a/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml new file mode 100644 index 00000000000..e69d16ff17f --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/conf2/conf/solrconfig.xml @@ -0,0 +1,68 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + explicit + true + text + + + + + + + + + + + + + + + diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index bea5d0c1104..2e908ef7458 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -17,17 +17,6 @@ package org.apache.solr.pkg; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - import org.apache.commons.codec.digest.DigestUtils; import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory; import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; @@ -40,11 +29,7 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.RequestWriter; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.request.*; import org.apache.solr.client.solrj.request.beans.Package; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; @@ -75,13 +60,22 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; import static org.apache.solr.core.TestSolrConfigHandler.getFileContent; -import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile; -import static org.apache.solr.filestore.TestDistribPackageStore.readFile; -import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey; +import static org.apache.solr.filestore.TestDistribPackageStore.*; @LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") public class TestPackages extends SolrCloudTestCase { @@ -102,6 +96,69 @@ public class TestPackages extends SolrCloudTestCase { @JsonProperty("class") public String klass; } + + + public void testCoreReloadingPlugin() throws Exception { + MiniSolrCloudCluster cluster = + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("conf2")) + .configure(); + try { + String FILE1 = "/mypkg/runtimelibs.jar"; + String COLLECTION_NAME = "testCoreReloadingPluginColl"; + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster); + postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, + "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + + Package.AddVersion add = new Package.AddVersion(); + add.version = "1.0"; + add.pkg = "mypkg"; + add.files = Arrays.asList(new String[]{FILE1}); + V2Request req = new V2Request.Builder("/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + + req.process(cluster.getSolrClient()); + TestDistribPackageStore.assertResponseValues(10, + () -> new V2Request.Builder("/cluster/package"). + withMethod(SolrRequest.METHOD.GET) + .build().process(cluster.getSolrClient()), + Utils.makeMap( + ":result:packages:mypkg[0]:version", "1.0", + ":result:packages:mypkg[0]:files[0]", FILE1 + )); + + CollectionAdminRequest + .createCollection(COLLECTION_NAME, "conf", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + verifyComponent(cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", add.pkg, add.version); + + + add.version = "2.0"; + req.process(cluster.getSolrClient()); + TestDistribPackageStore.assertResponseValues(10, + () -> new V2Request.Builder("/cluster/package"). + withMethod(SolrRequest.METHOD.GET) + .build().process(cluster.getSolrClient()), + Utils.makeMap( + ":result:packages:mypkg[1]:version", "2.0", + ":result:packages:mypkg[1]:files[0]", FILE1 + )); + new UpdateRequest().commit(cluster.getSolrClient(), COLLECTION_NAME); + + verifyComponent(cluster.getSolrClient(), + COLLECTION_NAME, "query", "filterCache", + "mypkg", "2.0" ); + } finally { + cluster.shutdown(); + } + } @Test @SuppressWarnings({"unchecked"}) public void testPluginLoading() throws Exception {