From 98f08d39aa723764a4bf39cfa5e358886e18a800 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Thu, 24 Oct 2019 08:55:11 +1100 Subject: [PATCH] SOLR-13822: Isolated Classloading from packages (#957) SOLR-13822: A Package management system with the following features. A packages.json in ZK to store the configuration, APIs to read/edit them and isolated classloaders to load the classes from those packages if the 'class' attribute is prefixed with `:` --- solr/CHANGES.txt | 8 +- .../org/apache/solr/api/AnnotatedApi.java | 4 + .../org/apache/solr/core/CoreContainer.java | 28 +- .../java/org/apache/solr/core/PluginBag.java | 28 +- .../java/org/apache/solr/core/PluginInfo.java | 44 +- .../org/apache/solr/core/RequestParams.java | 9 + .../java/org/apache/solr/core/SolrCore.java | 22 +- .../apache/solr/core/SolrResourceLoader.java | 334 ++++++------ .../solr/filestore/DistribPackageStore.java | 13 +- .../solr/filestore/PackageStoreAPI.java | 2 +- .../solr/handler/SolrConfigHandler.java | 39 +- .../solr/handler/component/SearchHandler.java | 60 ++- .../java/org/apache/solr/pkg/PackageAPI.java | 383 ++++++++++++++ .../org/apache/solr/pkg/PackageListeners.java | 111 ++++ .../org/apache/solr/pkg/PackageLoader.java | 276 ++++++++++ .../apache/solr/pkg/PackagePluginHolder.java | 123 +++++ .../solr/security/PermissionNameProvider.java | 3 + .../UpdateRequestProcessorChain.java | 43 +- .../org/apache/solr/pkg/TestPackages.java | 495 ++++++++++++++++++ .../solr/update/processor/RuntimeUrp.java | 2 +- .../client/solrj/request/beans/Package.java | 46 ++ .../solrj/request/beans/pakage-info.java | 23 + .../solr/common/cloud/SolrZkClient.java | 9 +- .../solr/common/cloud/ZkStateReader.java | 1 + .../solr/common/util/ReflectMapWriter.java | 2 +- 25 files changed, 1868 insertions(+), 240 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageAPI.java create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageListeners.java create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackageLoader.java create mode 100644 solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java create mode 100644 solr/core/src/test/org/apache/solr/pkg/TestPackages.java create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/pakage-info.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a87bbca54c6..14b96824234 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -100,7 +100,11 @@ Upgrade Notes New Features --------------------- -(No changes) +* SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya) + +* SOLR-13822: A Package management system with the following features. A packages.json in ZK to store + the configuration, APIs to read/edit them and isolated classloaders to load the classes from + hose packages if the 'class' attribute is prefixed with `:` (noble, Ishan Chattopadhyaya) Improvements --------------------- @@ -194,8 +198,6 @@ New Features * SOLR-8241: Add CaffeineCache, an efficient implementation of SolrCache.(Ben Manes, Shawn Heisey, David Smiley, Andrzej Bialecki) -* SOLR-13821: A Package store to store and load package artefacts (noble, Ishan Chattopadhyaya) - * SOLR-13298: Allow zplot to plot matrices (Joel Bernstein) Improvements diff --git a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java index e9073ae7bb1..a6ae1be6ebf 100644 --- a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java +++ b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java @@ -227,6 +227,7 @@ public class AnnotatedApi extends Api implements PermissionNameProvider { } if (isWrappedInPayloadObj) { PayloadObj payloadObj = new PayloadObj<>(cmd.name, cmd.getCommandData(), o); + cmd = payloadObj; method.invoke(obj, req, rsp, payloadObj); } else { method.invoke(obj, req, rsp, o); @@ -239,10 +240,13 @@ public class AnnotatedApi extends Api implements PermissionNameProvider { } catch (SolrException se) { + log.error("Error executing command ", se); throw se; } catch (InvocationTargetException ite) { + log.error("Error executing command ", ite); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ite.getCause()); } catch (Exception e) { + log.error("Error executing command : ", e); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 35fff633b45..49999085be2 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -101,6 +101,7 @@ import org.apache.solr.metrics.SolrCoreMetricManager; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.search.SolrFieldCacheBean; @@ -224,6 +225,7 @@ public class CoreContainer { protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler; private PackageStoreAPI packageStoreAPI; + private PackageLoader packageLoader; // Bits for the state variable. @@ -582,6 +584,10 @@ public class CoreContainer { return replayUpdatesExecutor; } + public PackageLoader getPackageLoader() { + return packageLoader; + } + public PackageStoreAPI getPackageStoreAPI() { return packageStoreAPI; } @@ -736,8 +742,12 @@ public class CoreContainer { if (isZooKeeperAware()) { metricManager.loadClusterReporters(metricReporters, this); + packageLoader = new PackageLoader(this); + containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP); + containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP); } + // setup executor to load cores in parallel ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService( ExecutorUtil.newMDCAwareFixedThreadPool( @@ -1210,15 +1220,15 @@ public class CoreContainer { * that calls solrCores.waitAddPendingCoreOps(...) and solrCores.removeFromPendingOps(...) * * 
-   *                                           
-   *                                           try {
-   *                                              solrCores.waitAddPendingCoreOps(dcore.getName());
-   *                                              createFromDescriptor(...);
-   *                                           } finally {
-   *                                              solrCores.removeFromPendingOps(dcore.getName());
-   *                                           }
-   *                                           
-   *
+ * + * try { + * solrCores.waitAddPendingCoreOps(dcore.getName()); + * createFromDescriptor(...); + * } finally { + * solrCores.removeFromPendingOps(dcore.getName()); + * } + * + * *

* Trying to put the waitAddPending... in this method results in Bad Things Happening due to race conditions. * getCore() depends on getting the core returned _if_ it's in the pending list due to some other thread opening it. diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index fa2c3e30b7b..f547d10ded4 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -43,8 +44,10 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.util.StrUtils; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.component.SearchComponent; +import org.apache.solr.pkg.PackagePluginHolder; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.update.processor.UpdateRequestProcessorChain; +import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder; import org.apache.solr.update.processor.UpdateRequestProcessorFactory; import org.apache.solr.util.CryptoKeys; import org.apache.solr.util.SimplePostTool; @@ -97,7 +100,7 @@ public class PluginBag implements AutoCloseable { this(klass, core, false); } - static void initInstance(Object inst, PluginInfo info) { + public static void initInstance(Object inst, PluginInfo info) { if (inst instanceof PluginInfoInitialized) { ((PluginInfoInitialized) inst).init(info); } else if (inst instanceof NamedListInitializedPlugin) { @@ -138,14 +141,23 @@ public class PluginBag implements AutoCloseable { log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name); return new LazyPluginHolder(meta, info, core, core.getResourceLoader(), false); } else { - T inst = core.createInstance(info.className, (Class) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader()); - initInstance(inst, info); - return new PluginHolder<>(info, inst); + if (info.pkgName != null) { + PackagePluginHolder holder = new PackagePluginHolder<>(info, core, meta); + return meta.clazz == UpdateRequestProcessorFactory.class ? + new PluginHolder(info, new LazyUpdateProcessorFactoryHolder(holder)) : + holder; + } else { + T inst = core.createInstance(info.className, (Class) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader(info.pkgName)); + initInstance(inst, info); + return new PluginHolder<>(info, inst); + } } } - /** make a plugin available in an alternate name. This is an internal API and not for public use - * @param src key in which the plugin is already registered + /** + * make a plugin available in an alternate name. This is an internal API and not for public use + * + * @param src key in which the plugin is already registered * @param target the new key in which the plugin should be aliased to. If target exists already, the alias fails * @return flag if the operation is successful or not */ @@ -340,8 +352,8 @@ public class PluginBag implements AutoCloseable { * An indirect reference to a plugin. It just wraps a plugin instance. * subclasses may choose to lazily load the plugin */ - public static class PluginHolder implements AutoCloseable { - private T inst; + public static class PluginHolder implements Supplier, AutoCloseable { + protected T inst; protected final PluginInfo pluginInfo; boolean registerAPI = false; diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java index 1bc85aeb0cd..bb290e12e81 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java +++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java @@ -16,14 +16,20 @@ */ package org.apache.solr.core; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import org.apache.solr.common.MapSerializable; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.Pair; import org.apache.solr.util.DOMUtil; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import java.util.*; - import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; @@ -35,27 +41,51 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME; * */ public class PluginInfo implements MapSerializable { - public final String name, className, type; + public final String name, className, type, pkgName; public final NamedList initArgs; public final Map attributes; public final List children; private boolean isFromSolrConfig; + + public PluginInfo(String type, Map attrs, NamedList initArgs, List children) { this.type = type; this.name = attrs.get(NAME); - this.className = attrs.get(CLASS_NAME); + Pair parsed = parseClassName(attrs.get(CLASS_NAME)); + this.className = parsed.second(); + this.pkgName = parsed.first(); this.initArgs = initArgs; attributes = unmodifiableMap(attrs); this.children = children == null ? Collections.emptyList(): unmodifiableList(children); isFromSolrConfig = false; } + /** class names can be prefixed with package name e.g: my_package:my.pkg.Class + * This checks if it is a package name prefixed classname. + * the return value has first = package name & second = class name + */ + static Pair parseClassName(String name) { + String pkgName = null; + String className = name; + if (name != null) { + int colonIdx = name.indexOf(':'); + if (colonIdx > -1) { + pkgName = name.substring(0, colonIdx); + className = name.substring(colonIdx + 1); + } + } + return new Pair<>(pkgName, className); + + } + public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) { type = node.getNodeName(); name = DOMUtil.getAttr(node, NAME, requireName ? err : null); - className = DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null); + Pair parsed = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null)); + className = parsed.second(); + pkgName = parsed.first(); initArgs = DOMUtil.childNodesToNamedList(node); attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes())); children = loadSubPlugins(node); @@ -85,7 +115,9 @@ public class PluginInfo implements MapSerializable { } this.type = type; this.name = (String) m.get(NAME); - this.className = (String) m.get(CLASS_NAME); + Pair parsed = parseClassName((String) m.get(CLASS_NAME)); + this.className = parsed.second(); + this.pkgName = parsed.first(); attributes = unmodifiableMap(m); this.children = Collections.emptyList(); isFromSolrConfig = true; diff --git a/solr/core/src/java/org/apache/solr/core/RequestParams.java b/solr/core/src/java/org/apache/solr/core/RequestParams.java index 50a4fd0de78..d1f7f3d295e 100644 --- a/solr/core/src/java/org/apache/solr/core/RequestParams.java +++ b/solr/core/src/java/org/apache/solr/core/RequestParams.java @@ -250,9 +250,18 @@ public class RequestParams implements MapSerializable { return m1; } + /** + * @param type one of defaults, appends, invariants + */ public VersionedParams getParams(String type) { return paramsMap.get(type); } + + /**get the raw map + */ + public Map get() { + return defaults; + } } public static class VersionedParams extends MapSolrParams { diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index ef681be2c25..09067b18978 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -110,6 +110,8 @@ import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.metrics.SolrCoreMetricManager; import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; +import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.BinaryResponseWriter; @@ -238,6 +240,8 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab public volatile boolean indexEnabled = true; public volatile boolean readOnly = false; + private PackageListeners packageListeners = new PackageListeners(this); + public Set getMetricNames() { return metricNames; } @@ -262,6 +266,10 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab return restManager; } + public PackageListeners getPackageListeners() { + return packageListeners; + } + static int boolean_query_max_clause_count = Integer.MIN_VALUE; private ExecutorService coreAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Async Task"); @@ -275,6 +283,18 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab return resourceLoader; } + /** Gets the SolrResourceLoader for a given package + * @param pkg The package name + */ + public SolrResourceLoader getResourceLoader(String pkg) { + if (pkg == null) { + return resourceLoader; + } + PackageLoader.Package aPackage = coreContainer.getPackageLoader().getPackage(pkg); + PackageLoader.Package.Version latest = aPackage.getLatest(); + return latest.getLoader(); + } + /** * Gets the configuration resource name used by this core instance. * @@ -857,7 +877,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab public T createInitInstance(PluginInfo info, Class cast, String msg, String defClassName) { if (info == null) return null; - T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader()); + T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName)); if (o instanceof PluginInfoInitialized) { ((PluginInfoInitialized) o).init(info); } else if (o instanceof NamedListInitializedPlugin) { diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index 41329189509..a57660eb980 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.invoke.MethodHandles; import java.lang.reflect.Constructor; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.nio.charset.CharacterCodingException; @@ -81,9 +82,8 @@ import org.slf4j.LoggerFactory; /** * @since solr 1.3 - */ -public class SolrResourceLoader implements ResourceLoader,Closeable -{ + */ +public class SolrResourceLoader implements ResourceLoader, Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); static final String project = "solr"; @@ -96,31 +96,33 @@ public class SolrResourceLoader implements ResourceLoader,Closeable }; private static final java.lang.String SOLR_CORE_NAME = "solr.core.name"; private static Set loggedOnce = new ConcurrentSkipListSet<>(); + private static final Charset UTF_8 = StandardCharsets.UTF_8; + + private String name = ""; protected URLClassLoader classLoader; private final Path instanceDir; private String dataDir; - + private final List waitingForCore = Collections.synchronizedList(new ArrayList()); private final List infoMBeans = Collections.synchronizedList(new ArrayList()); private final List waitingForResources = Collections.synchronizedList(new ArrayList()); - private static final Charset UTF_8 = StandardCharsets.UTF_8; private final Properties coreProperties; private volatile boolean live; - + // Provide a registry so that managed resources can register themselves while the XML configuration // documents are being parsed ... after all are registered, they are asked by the RestManager to // initialize themselves. This two-step process is required because not all resources are available // (such as the SolrZkClient) when XML docs are being parsed. private RestManager.Registry managedResourceRegistry; - + public synchronized RestManager.Registry getManagedResourceRegistry() { if (managedResourceRegistry == null) { - managedResourceRegistry = new RestManager.Registry(); + managedResourceRegistry = new RestManager.Registry(); } - return managedResourceRegistry; + return managedResourceRegistry; } public SolrResourceLoader() { @@ -134,11 +136,20 @@ public class SolrResourceLoader implements ResourceLoader,Closeable * found in the "lib/" directory in the specified instance directory. * If the instance directory is not specified (=null), SolrResourceLoader#locateInstanceDir will provide one. */ - public SolrResourceLoader(Path instanceDir, ClassLoader parent) - { + public SolrResourceLoader(Path instanceDir, ClassLoader parent) { this(instanceDir, parent, null); } + public SolrResourceLoader(String name, List classpath, Path instanceDir, ClassLoader parent) throws MalformedURLException { + this(instanceDir, parent); + this.name = name; + for (Path path : classpath) { + addToClassLoader(path.toUri().normalize().toURL()); + } + + } + + public SolrResourceLoader(Path instanceDir) { this(instanceDir, null, null); } @@ -157,7 +168,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable if (instanceDir == null) { this.instanceDir = SolrResourceLoader.locateSolrHome().toAbsolutePath().normalize(); log.debug("new SolrResourceLoader for deduced Solr Home: '{}'", this.instanceDir); - } else{ + } else { this.instanceDir = instanceDir.toAbsolutePath().normalize(); log.debug("new SolrResourceLoader for directory: '{}'", this.instanceDir); } @@ -167,7 +178,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } this.classLoader = URLClassLoader.newInstance(new URL[0], parent); - /* + /* * Skip the lib subdirectory when we are loading from the solr home. * Otherwise load it, so core lib directories still get loaded. * The default sharedLib will pick this up later, and if the user has @@ -264,6 +275,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable /** * Utility method to get the URLs of all paths under a given directory that match a filter + * * @param libDir the root directory * @param filter the filter * @return all matching URLs @@ -296,8 +308,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable /** * Utility method to get the URLs of all paths under a given directory that match a regex + * * @param libDir the root directory - * @param regex the regex as a String + * @param regex the regex as a String * @return all matching URLs * @throws IOException on error */ @@ -310,15 +323,17 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } }); } - - /** Ensures a directory name always ends with a '/'. */ + + /** + * Ensures a directory name always ends with a '/'. + */ public static String normalizeDir(String path) { - return ( path != null && (!(path.endsWith("/") || path.endsWith("\\"))) )? path + File.separator : path; + return (path != null && (!(path.endsWith("/") || path.endsWith("\\")))) ? path + File.separator : path; } - + public String[] listConfigDir() { File configdir = new File(getConfigDir()); - if( configdir.exists() && configdir.isDirectory() ) { + if (configdir.exists() && configdir.isDirectory()) { return configdir.list(); } else { return new String[0]; @@ -328,8 +343,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable public String getConfigDir() { return instanceDir.resolve("conf").toString(); } - - public String getDataDir() { + + public String getDataDir() { return dataDir; } @@ -341,23 +356,28 @@ public class SolrResourceLoader implements ResourceLoader,Closeable * EXPERT *

* The underlying class loader. Most applications will not need to use this. + * * @return The {@link ClassLoader} */ public ClassLoader getClassLoader() { return classLoader; } - /** Opens a schema resource by its name. + /** + * Opens a schema resource by its name. * Override this method to customize loading schema resources. - *@return the stream for the named schema + * + * @return the stream for the named schema */ public InputStream openSchema(String name) throws IOException { return openResource(name); } - - /** Opens a config resource by its name. + + /** + * Opens a config resource by its name. * Override this method to customize loading config resources. - *@return the stream for the named configuration + * + * @return the stream for the named configuration */ public InputStream openConfig(String name) throws IOException { return openResource(name); @@ -372,14 +392,16 @@ public class SolrResourceLoader implements ResourceLoader,Closeable throw new IOException("File " + pathToCheck + " is outside resource loader dir " + instanceDir + "; set -Dsolr.allow.unsafe.resourceloading=true to allow unsafe loading"); } - - /** Opens any resource by its name. + + /** + * Opens any resource by its name. * By default, this will look in multiple locations to load the resource: * $configDir/$resource (if resource is not absolute) * $CWD/$resource * otherwise, it will look for it in any jar accessible through the class loader. * Override this method to customize loading resources. - *@return the stream for the named resource + * + * @return the stream for the named resource */ @Override public InputStream openResource(String resource) throws IOException { @@ -461,22 +483,22 @@ public class SolrResourceLoader implements ResourceLoader,Closeable * @throws IOException If there is a low-level I/O error. */ public List getLines(String resource, - String encoding) throws IOException { + String encoding) throws IOException { return getLines(resource, Charset.forName(encoding)); } - public List getLines(String resource, Charset charset) throws IOException{ + public List getLines(String resource, Charset charset) throws IOException { try { return WordlistLoader.getLines(openResource(resource), charset); } catch (CharacterCodingException ex) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Error loading resource (wrong encoding?): " + resource, ex); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Error loading resource (wrong encoding?): " + resource, ex); } } /* - * A static map of short class name to fully qualified class name + * A static map of short class name to fully qualified class name */ private static final Map classNameCache = new ConcurrentHashMap<>(); @@ -486,14 +508,14 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } // Using this pattern, legacy analysis components from previous Solr versions are identified and delegated to SPI loader: - private static final Pattern legacyAnalysisPattern = - Pattern.compile("((\\Q"+base+".analysis.\\E)|(\\Q"+project+".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory"); + private static final Pattern legacyAnalysisPattern = + Pattern.compile("((\\Q" + base + ".analysis.\\E)|(\\Q" + project + ".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory"); @Override public Class findClass(String cname, Class expectedType) { return findClass(cname, expectedType, empty); } - + /** * This method loads a class either with its FQN or a short-name (solr.class-simplename or class-simplename). * It tries to load the class with the name that is given first and if it fails, it tries all the known @@ -501,25 +523,25 @@ public class SolrResourceLoader implements ResourceLoader,Closeable * for the same class faster. The caching is done only if the class is loaded by the webapp classloader and it * is loaded using a shortname. * - * @param cname The name or the short name of the class. + * @param cname The name or the short name of the class. * @param subpackages the packages to be tried if the cname starts with solr. * @return the loaded class. An exception is thrown if it fails */ public Class findClass(String cname, Class expectedType, String... subpackages) { if (subpackages == null || subpackages.length == 0 || subpackages == packages) { subpackages = packages; - String c = classNameCache.get(cname); - if(c != null) { + String c = classNameCache.get(cname); + if (c != null) { try { return Class.forName(c, true, classLoader).asSubclass(expectedType); } catch (ClassNotFoundException | ClassCastException e) { // this can happen if the legacyAnalysisPattern below caches the wrong thing - log.warn("Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e); + log.warn( name + " Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e); classNameCache.remove(cname); } } } - + Class clazz = null; try { // first try legacy analysis patterns, now replaced by Lucene's Analysis package: @@ -537,43 +559,43 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } else { log.warn("'{}' looks like an analysis factory, but caller requested different class type: {}", cname, expectedType.getName()); } - } catch (IllegalArgumentException ex) { + } catch (IllegalArgumentException ex) { // ok, we fall back to legacy loading } } - + // first try cname == full name try { return clazz = Class.forName(cname, true, classLoader).asSubclass(expectedType); } catch (ClassNotFoundException e) { - String newName=cname; + String newName = cname; if (newName.startsWith(project)) { - newName = cname.substring(project.length()+1); + newName = cname.substring(project.length() + 1); } for (String subpackage : subpackages) { try { String name = base + '.' + subpackage + newName; log.trace("Trying class name " + name); - return clazz = Class.forName(name,true,classLoader).asSubclass(expectedType); + return clazz = Class.forName(name, true, classLoader).asSubclass(expectedType); } catch (ClassNotFoundException e1) { // ignore... assume first exception is best. } } - - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error loading class '" + cname + "'", e); + + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, name +" Error loading class '" + cname + "'", e); } - + } finally { if (clazz != null) { //cache the shortname vs FQN if it is loaded by the webapp classloader and it is loaded // using a shortname if (clazz.getClassLoader() == SolrResourceLoader.class.getClassLoader() && - !cname.equals(clazz.getName()) && - (subpackages.length == 0 || subpackages == packages)) { + !cname.equals(clazz.getName()) && + (subpackages.length == 0 || subpackages == packages)) { //store in the cache classNameCache.put(cname, clazz.getName()); } - + // print warning if class is deprecated if (clazz.isAnnotationPresent(Deprecated.class)) { log.warn("Solr loaded a deprecated plugin/analysis class [{}]. Please consult documentation how to replace it accordingly.", @@ -582,9 +604,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } } } - + static final String empty[] = new String[0]; - + @Override public T newInstance(String name, Class expectedType) { return newInstance(name, expectedType, empty); @@ -593,33 +615,32 @@ public class SolrResourceLoader implements ResourceLoader,Closeable private static final Class[] NO_CLASSES = new Class[0]; private static final Object[] NO_OBJECTS = new Object[0]; - public T newInstance(String cname, Class expectedType, String ... subpackages) { + public T newInstance(String cname, Class expectedType, String... subpackages) { return newInstance(cname, expectedType, subpackages, NO_CLASSES, NO_OBJECTS); } - public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String ... subpackages) { + public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String... subpackages) { Class clazz = findClass(cname, CoreAdminHandler.class, subpackages); - if( clazz == null ) { - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Can not find class: "+cname + " in " + classLoader); + if (clazz == null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Can not find class: " + cname + " in " + classLoader); } - + CoreAdminHandler obj = null; try { Constructor ctor = clazz.getConstructor(CoreContainer.class); obj = ctor.newInstance(coreContainer); - } - catch (Exception e) { - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Error instantiating class: '" + clazz.getName()+"'", e); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Error instantiating class: '" + clazz.getName() + "'", e); } if (!live) { //TODO: Does SolrCoreAware make sense here since in a multi-core context // which core are we talking about ? - if( obj instanceof ResourceLoaderAware ) { - assertAwareCompatibility( ResourceLoaderAware.class, obj ); - waitingForResources.add( (ResourceLoaderAware)obj ); + if (obj instanceof ResourceLoaderAware) { + assertAwareCompatibility(ResourceLoaderAware.class, obj); + waitingForResources.add((ResourceLoaderAware) obj); } } @@ -627,12 +648,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } - - public T newInstance(String cName, Class expectedType, String [] subPackages, Class[] params, Object[] args){ + public T newInstance(String cName, Class expectedType, String[] subPackages, Class[] params, Object[] args) { Class clazz = findClass(cName, expectedType, subPackages); - if( clazz == null ) { - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Can not find class: "+cName + " in " + classLoader); + if (clazz == null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Can not find class: " + cName + " in " + classLoader); } T obj = null; @@ -653,25 +673,25 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } } catch (Error err) { - log.error("Loading Class " + cName + " ("+clazz.getName() + ") triggered serious java error: " - + err.getClass().getName(), err); + log.error("Loading Class " + cName + " (" + clazz.getName() + ") triggered serious java error: " + + err.getClass().getName(), err); throw err; } catch (Exception e) { - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Error instantiating class: '" + clazz.getName()+"'", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Error instantiating class: '" + clazz.getName() + "'", e); } if (!live) { - if( obj instanceof SolrCoreAware ) { - assertAwareCompatibility( SolrCoreAware.class, obj ); - waitingForCore.add( (SolrCoreAware)obj ); + if (obj instanceof SolrCoreAware) { + assertAwareCompatibility(SolrCoreAware.class, obj); + waitingForCore.add((SolrCoreAware) obj); } - if( obj instanceof ResourceLoaderAware ) { - assertAwareCompatibility( ResourceLoaderAware.class, obj ); - waitingForResources.add( (ResourceLoaderAware)obj ); + if (obj instanceof ResourceLoaderAware) { + assertAwareCompatibility(ResourceLoaderAware.class, obj); + waitingForResources.add((ResourceLoaderAware) obj); } - if (obj instanceof SolrInfoBean){ + if (obj instanceof SolrInfoBean) { //TODO: Assert here? infoMBeans.add((SolrInfoBean) obj); } @@ -680,12 +700,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable return obj; } - + /** * Tell all {@link SolrCoreAware} instances about the SolrCore */ - public void inform(SolrCore core) - { + public void inform(SolrCore core) { this.dataDir = core.getDataDir(); // make a copy to avoid potential deadlock of a callback calling newInstance and trying to @@ -698,22 +717,21 @@ public class SolrResourceLoader implements ResourceLoader,Closeable waitingForCore.clear(); } - for( SolrCoreAware aware : arr) { - aware.inform( core ); + for (SolrCoreAware aware : arr) { + aware.inform(core); } } // this is the last method to be called in SolrCore before the latch is released. live = true; } - + /** * Tell all {@link ResourceLoaderAware} instances about the loader */ - public void inform( ResourceLoader loader ) throws IOException - { + public void inform(ResourceLoader loader) throws IOException { - // make a copy to avoid potential deadlock of a callback adding to the list + // make a copy to avoid potential deadlock of a callback adding to the list ResourceLoaderAware[] arr; while (waitingForResources.size() > 0) { @@ -722,7 +740,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable waitingForResources.clear(); } - for( ResourceLoaderAware aware : arr) { + for (ResourceLoaderAware aware : arr) { aware.inform(loader); } } @@ -730,6 +748,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable /** * Register any {@link SolrInfoBean}s + * * @param infoRegistry The Info Registry */ public void inform(Map infoRegistry) { @@ -755,7 +774,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } } } - + /** * Determines the solrhome from the environment. * Tries JNDI (java:comp/env/solr/home) then system property (solr.solr.home); @@ -765,12 +784,13 @@ public class SolrResourceLoader implements ResourceLoader,Closeable /** * Finds the solrhome based on looking up the value in one of three places: *

    - *
  1. JNDI: via java:comp/env/solr/home
    2. - *
  2. The system property solr.solr.home
    3. - *
  3. Look in the current working directory for a solr/ directory
    4. + *
  4. JNDI: via java:comp/env/solr/home
    5. + *
  5. The system property solr.solr.home
    6. + *
  6. Look in the current working directory for a solr/ directory
    7. *
- * + *

* The return value is normalized. Normalization essentially means it ends in a trailing slash. + * * @return A normalized solrhome * @see #normalizeDir(String) */ @@ -780,27 +800,27 @@ public class SolrResourceLoader implements ResourceLoader,Closeable // Try JNDI try { Context c = new InitialContext(); - home = (String)c.lookup("java:comp/env/"+project+"/home"); - logOnceInfo("home_using_jndi", "Using JNDI solr.home: "+home ); + home = (String) c.lookup("java:comp/env/" + project + "/home"); + logOnceInfo("home_using_jndi", "Using JNDI solr.home: " + home); } catch (NoInitialContextException e) { - log.debug("JNDI not configured for "+project+" (NoInitialContextEx)"); + log.debug("JNDI not configured for " + project + " (NoInitialContextEx)"); } catch (NamingException e) { - log.debug("No /"+project+"/home in JNDI"); - } catch( RuntimeException ex ) { + log.debug("No /" + project + "/home in JNDI"); + } catch (RuntimeException ex) { log.warn("Odd RuntimeException while testing for JNDI: " + ex.getMessage()); - } - + } + // Now try system property - if( home == null ) { + if (home == null) { String prop = project + ".solr.home"; home = System.getProperty(prop); - if( home != null ) { - logOnceInfo("home_using_sysprop", "Using system property "+prop+": " + home ); + if (home != null) { + logOnceInfo("home_using_sysprop", "Using system property " + prop + ": " + home); } } - + // if all else fails, try - if( home == null ) { + if (home == null) { home = project + '/'; logOnceInfo("home_default", project + " home defaulted to '" + home + "' (could not find system property or JNDI)"); } @@ -809,22 +829,23 @@ public class SolrResourceLoader implements ResourceLoader,Closeable /** * Solr allows users to store arbitrary files in a special directory located directly under SOLR_HOME. - * + *

* This directory is generally created by each node on startup. Files located in this directory can then be * manipulated using select Solr features (e.g. streaming expressions). */ public static final String USER_FILES_DIRECTORY = "userfiles"; + public static void ensureUserFilesDataDir(Path solrHome) { final Path userFilesPath = getUserFilesPath(solrHome); final File userFilesDirectory = new File(userFilesPath.toString()); - if (! userFilesDirectory.exists()) { + if (!userFilesDirectory.exists()) { try { final boolean created = userFilesDirectory.mkdir(); - if (! created) { + if (!created) { log.warn("Unable to create [{}] directory in SOLR_HOME [{}]. Features requiring this directory may fail.", USER_FILES_DIRECTORY, solrHome); } } catch (Exception e) { - log.warn("Unable to create [" + USER_FILES_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e); + log.warn("Unable to create [" + USER_FILES_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e); } } } @@ -847,72 +868,73 @@ public class SolrResourceLoader implements ResourceLoader,Closeable public Path getInstancePath() { return instanceDir; } - + /** * Keep a list of classes that are allowed to implement each 'Aware' interface */ private static final Map awareCompatibility; + static { awareCompatibility = new HashMap<>(); - awareCompatibility.put( - SolrCoreAware.class, new Class[] { - // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY - // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311 - CodecFactory.class, - DirectoryFactory.class, - ManagedIndexSchemaFactory.class, - QueryResponseWriter.class, - SearchComponent.class, - ShardHandlerFactory.class, - SimilarityFactory.class, - SolrRequestHandler.class, - UpdateRequestProcessorFactory.class - } + awareCompatibility.put( + SolrCoreAware.class, new Class[]{ + // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY + // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311 + CodecFactory.class, + DirectoryFactory.class, + ManagedIndexSchemaFactory.class, + QueryResponseWriter.class, + SearchComponent.class, + ShardHandlerFactory.class, + SimilarityFactory.class, + SolrRequestHandler.class, + UpdateRequestProcessorFactory.class + } ); awareCompatibility.put( - ResourceLoaderAware.class, new Class[] { - // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY - // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311 - CharFilterFactory.class, - TokenFilterFactory.class, - TokenizerFactory.class, - QParserPlugin.class, - FieldType.class - } + ResourceLoaderAware.class, new Class[]{ + // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY + // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311 + CharFilterFactory.class, + TokenFilterFactory.class, + TokenizerFactory.class, + QParserPlugin.class, + FieldType.class + } ); } /** * Utility function to throw an exception if the class is invalid */ - static void assertAwareCompatibility( Class aware, Object obj ) - { - Class[] valid = awareCompatibility.get( aware ); - if( valid == null ) { - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Unknown Aware interface: "+aware ); + static void assertAwareCompatibility(Class aware, Object obj) { + Class[] valid = awareCompatibility.get(aware); + if (valid == null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unknown Aware interface: " + aware); } - for( Class v : valid ) { - if( v.isInstance( obj ) ) { + for (Class v : valid) { + if (v.isInstance(obj)) { return; } } StringBuilder builder = new StringBuilder(); - builder.append( "Invalid 'Aware' object: " ).append( obj ); - builder.append( " -- ").append( aware.getName() ); - builder.append( " must be an instance of: " ); - for( Class v : valid ) { - builder.append( "[" ).append( v.getName() ).append( "] ") ; + builder.append("Invalid 'Aware' object: ").append(obj); + builder.append(" -- ").append(aware.getName()); + builder.append(" must be an instance of: "); + for (Class v : valid) { + builder.append("[").append(v.getName()).append("] "); } - throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, builder.toString() ); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, builder.toString()); } @Override public void close() throws IOException { IOUtils.close(classLoader); } - public List getInfoMBeans(){ + + public List getInfoMBeans() { return Collections.unmodifiableList(infoMBeans); } @@ -922,8 +944,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable File confFile = new File(loader.getConfigDir(), resourceName); try { File parentDir = confFile.getParentFile(); - if ( ! parentDir.isDirectory()) { - if ( ! parentDir.mkdirs()) { + if (!parentDir.isDirectory()) { + if (!parentDir.mkdirs()) { final String msg = "Can't create managed schema directory " + parentDir.getAbsolutePath(); log.error(msg); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg); diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java index cd50d934be4..cd653e7a6ee 100644 --- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java @@ -354,26 +354,21 @@ public class DistribPackageStore implements PackageStore { Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), url, null); } catch (Exception e) { log.info("Node: " + node + - " failed to respond for blob notification", e); + " failed to respond for file fetch notification", e); //ignore the exception // some nodes may be down or not responding } i++; } } finally { - new Thread(() -> { + coreContainer.getUpdateShardHandler().getUpdateExecutor().submit(() -> { try { - // keep the jar in memory for 10 secs , so that - //every node can download it from memory without the file system Thread.sleep(10 * 1000); - } catch (Exception e) { - //don't care } finally { tmpFiles.remove(entry.getPath()); } - }).start(); - - + return null; + }); } } diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java index a51b3662951..7e80b9a4475 100644 --- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java @@ -198,7 +198,7 @@ public class PackageStoreAPI { cryptoKeys = new CryptoKeys(keys); } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Error parsing public keyts in ZooKeeper"); + "Error parsing public keys in ZooKeeper"); } for (String sig : sigs) { if (cryptoKeys.verify(sig, buf) == null) { diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index 5952d80f29d..004da318ae3 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -68,6 +68,7 @@ import org.apache.solr.core.RequestParams; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.pkg.PackageListeners; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; @@ -150,7 +151,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa public static boolean getImmutable(SolrCore core) { NamedList configSetProperties = core.getConfigSetProperties(); - if(configSetProperties == null) return false; + if (configSetProperties == null) return false; Object immutable = configSetProperties.get(IMMUTABLE_CONFIGSET_ARG); return immutable != null && Boolean.parseBoolean(immutable.toString()); } @@ -245,10 +246,26 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa if (componentName != null) { Map map = (Map) val.get(parts.get(1)); if (map != null) { - val.put(parts.get(1), makeMap(componentName, map.get(componentName))); + Object o = map.get(componentName); + val.put(parts.get(1), makeMap(componentName, o)); + if (req.getParams().getBool("meta", false)) { + // meta=true is asking for the package info of the plugin + // We go through all the listeners and see if there is one registered for this plugin + List listeners = req.getCore().getPackageListeners().getListeners(); + for (PackageListeners.Listener listener : + listeners) { + PluginInfo info = listener.pluginInfo(); + if(info == null) continue; + if (info.type.equals(parts.get(1)) && info.name.equals(componentName)) { + if (o instanceof Map) { + Map m1 = (Map) o; + m1.put("_packageinfo_", listener.getPackageVersion()); + } + } + } + } } } - resp.add("config", val); } } @@ -429,7 +446,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa List errs = CommandOperation.captureErrors(ops); if (!errs.isEmpty()) { - throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing params", errs); + throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, "error processing params", errs); } SolrResourceLoader loader = req.getCore().getResourceLoader(); @@ -492,14 +509,15 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa } List errs = CommandOperation.captureErrors(ops); if (!errs.isEmpty()) { - throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing commands", errs); + log.error("ERROR:" + Utils.toJSONString(errs)); + throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, "error processing commands", errs); } SolrResourceLoader loader = req.getCore().getResourceLoader(); if (loader instanceof ZkSolrResourceLoader) { int latestVersion = ZkController.persistConfigResourceToZooKeeper((ZkSolrResourceLoader) loader, overlay.getZnodeVersion(), ConfigOverlay.RESOURCE_NAME, overlay.toByteArray(), true); - log.info("Executed config commands successfully and persisted to ZK {}", ops); + log.debug("Executed config commands successfully and persisted to ZK {}", ops); waitForAllReplicasState(req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(), req.getCore().getCoreContainer().getZkController(), ConfigOverlay.NAME, @@ -530,8 +548,8 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa op.getMap(PluginInfo.INVARIANTS, null); op.getMap(PluginInfo.APPENDS, null); if (op.hasError()) return overlay; - if(info.clazz == PluginBag.RuntimeLib.class) { - if(!PluginBag.RuntimeLib.isEnabled()){ + if (info.clazz == PluginBag.RuntimeLib.class) { + if (!PluginBag.RuntimeLib.isEnabled()) { op.addError("Solr not started with -Denable.runtime.lib=true"); return overlay; } @@ -563,7 +581,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa private boolean pluginExists(SolrConfig.SolrPluginInfo info, ConfigOverlay overlay, String name) { List l = req.getCore().getSolrConfig().getPluginInfos(info.clazz.getName()); - for (PluginInfo pluginInfo : l) if(name.equals( pluginInfo.name)) return true; + for (PluginInfo pluginInfo : l) if (name.equals(pluginInfo.name)) return true; return overlay.getNamedPlugins(info.getCleanTag()).containsKey(name); } @@ -574,6 +592,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa try { req.getCore().createInitInstance(new PluginInfo(SolrRequestHandler.TYPE, op.getDataMap()), expected, clz, ""); } catch (Exception e) { + log.error("Error checking plugin : ", e); op.addError(e.getMessage()); return false; } @@ -679,7 +698,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa c == '_' || c == '-' || c == '.' - ) continue; + ) continue; else { return formatString("''{0}'' name should only have chars [a-zA-Z_-.0-9] ", s); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index 64b8c9a441d..f617fcbb0a7 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -40,6 +40,8 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.SolrQueryTimeoutImpl; @@ -58,11 +60,9 @@ import static org.apache.solr.common.params.CommonParams.PATH; /** - * * Refer SOLR-281 - * */ -public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , PluginInfoInitialized, PermissionNameProvider { +public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, PluginInfoInitialized, PermissionNameProvider { static final String INIT_COMPONENTS = "components"; static final String INIT_FIRST_COMPONENTS = "first-components"; static final String INIT_LAST_COMPONENTS = "last-components"; @@ -70,22 +70,21 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected volatile List components; - private ShardHandlerFactory shardHandlerFactory ; + private ShardHandlerFactory shardHandlerFactory; private PluginInfo shfInfo; private SolrCore core; - protected List getDefaultComponents() - { + protected List getDefaultComponents() { ArrayList names = new ArrayList<>(8); - names.add( QueryComponent.COMPONENT_NAME ); - names.add( FacetComponent.COMPONENT_NAME ); - names.add( FacetModule.COMPONENT_NAME ); - names.add( MoreLikeThisComponent.COMPONENT_NAME ); - names.add( HighlightComponent.COMPONENT_NAME ); - names.add( StatsComponent.COMPONENT_NAME ); - names.add( DebugComponent.COMPONENT_NAME ); - names.add( ExpandComponent.COMPONENT_NAME); - names.add( TermsComponent.COMPONENT_NAME); + names.add(QueryComponent.COMPONENT_NAME); + names.add(FacetComponent.COMPONENT_NAME); + names.add(FacetModule.COMPONENT_NAME); + names.add(MoreLikeThisComponent.COMPONENT_NAME); + names.add(HighlightComponent.COMPONENT_NAME); + names.add(StatsComponent.COMPONENT_NAME); + names.add(DebugComponent.COMPONENT_NAME); + names.add(ExpandComponent.COMPONENT_NAME); + names.add(TermsComponent.COMPONENT_NAME); return names; } @@ -94,7 +93,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , public void init(PluginInfo info) { init(info.initArgs); for (PluginInfo child : info.children) { - if("shardHandlerFactory".equals(child.type)){ + if ("shardHandlerFactory".equals(child.type)) { this.shfInfo = child; break; } @@ -113,8 +112,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , */ @Override @SuppressWarnings("unchecked") - public void inform(SolrCore core) - { + public void inform(SolrCore core) { this.core = core; List c = (List) initArgs.get(INIT_COMPONENTS); Set missing = new HashSet<>(core.getSearchComponents().checkContains(c)); @@ -143,6 +141,32 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , }); } + if (core.getCoreContainer().isZooKeeperAware()) { + core.getPackageListeners().addListener(new PackageListeners.Listener() { + @Override + public String packageName() { + return null; + } + + @Override + public PluginInfo pluginInfo() { + return null; + } + + @Override + public void changed(PackageLoader.Package pkg) { + //we could optimize this by listening to only relevant packages, + // but it is not worth optimizing as these are lightweight objects + components = null; + } + + @Override + public PackageLoader.Package.Version getPackageVersion() { + return null; + } + }); + } + } private void initComponents() { diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java new file mode 100644 index 00000000000..f202503a87e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.solr.api.Command; +import org.apache.solr.api.EndPoint; +import org.apache.solr.api.PayloadObj; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.request.beans.Package; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.util.CommandOperation; +import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.filestore.PackageStoreAPI; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; + +public class PackageAPI { + public static final String PACKAGES = "packages"; + public final boolean enablePackages = Boolean.parseBoolean(System.getProperty("enable.packages", "false")); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + final CoreContainer coreContainer; + private ObjectMapper mapper = new ObjectMapper(); + private final PackageLoader packageLoader; + Packages pkgs; + + public final Edit editAPI = new Edit(); + public final Read readAPI = new Read(); + + public PackageAPI(CoreContainer coreContainer, PackageLoader loader) { + this.coreContainer = coreContainer; + this.packageLoader = loader; + pkgs = new Packages(); + SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); + try { + registerListener(zkClient); + } catch (KeeperException | InterruptedException e) { + SolrZkClient.checkInterrupted(e); + } + } + + private void registerListener(SolrZkClient zkClient) + throws KeeperException, InterruptedException { + String path = SOLR_PKGS_PATH; + zkClient.exists(path, + new Watcher() { + + @Override + public void process(WatchedEvent event) { + // session events are not change events, and do not remove the watcher + if (Event.EventType.None.equals(event.getType())) { + return; + } + try { + synchronized (this) { + log.debug("Updating [{}] ... ", path); + + // remake watch + final Watcher thisWatch = this; + final Stat stat = new Stat(); + final byte[] data = zkClient.getData(path, thisWatch, stat, true); + pkgs = readPkgsFromZk(data, stat); + packageLoader.refreshPackageConf(); + } + } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { + log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: [{}]", e.getMessage()); + } catch (KeeperException e) { + log.error("A ZK error has occurred", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); + } catch (InterruptedException e) { + // Restore the interrupted status + Thread.currentThread().interrupt(); + log.warn("Interrupted", e); + } + } + + }, true); + } + + + private Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { + + if (data == null || stat == null) { + stat = new Stat(); + data = coreContainer.getZkController().getZkClient() + .getData(ZkStateReader.CLUSTER_PROPS, null, stat, true); + + } + Packages packages = null; + if (data == null || data.length == 0) { + packages = new Packages(); + } else { + try { + packages = mapper.readValue(data, Packages.class); + packages.znodeVersion = stat.getVersion(); + } catch (IOException e) { + //invalid data in packages + //TODO handle properly; + return new Packages(); + } + } + return packages; + } + + + public static class Packages implements ReflectMapWriter { + @JsonProperty + public int znodeVersion = -1; + + @JsonProperty + public Map> packages = new LinkedHashMap<>(); + + + public Packages copy() { + Packages p = new Packages(); + p.znodeVersion = this.znodeVersion; + p.packages = new LinkedHashMap<>(); + packages.forEach((s, versions) -> + p.packages.put(s, new ArrayList<>(versions))); + return p; + } + } + + public static class PkgVersion implements ReflectMapWriter { + + @JsonProperty + public String version; + + @JsonProperty + public List files; + + public PkgVersion() { + } + + public PkgVersion(Package.AddVersion addVersion) { + this.version = addVersion.version; + this.files = addVersion.files; + } + + + @Override + public boolean equals(Object obj) { + if (obj instanceof PkgVersion) { + PkgVersion that = (PkgVersion) obj; + return Objects.equals(this.version, that.version) + && Objects.equals(this.files, that.files); + + } + return false; + } + } + + + @EndPoint(method = SolrRequest.METHOD.POST, + path = "/cluster/package", + permission = PACKAGE_EDIT_PERM) + public class Edit { + + @Command(name = "refresh") + public void refresh(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj payload) { + String p = payload.get(); + if (p == null) { + payload.addError("Package null"); + return; + } + PackageLoader.Package pkg = coreContainer.getPackageLoader().getPackage(p); + if (pkg == null) { + payload.addError("No such package: " + p); + return; + } + + for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) { + Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), + coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&refreshPackage=" + p, + Utils.JAVABINCONSUMER); + } + + + } + + + @Command(name = "add") + public void add(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj payload) { + if (!checkEnabled(payload)) return; + Package.AddVersion add = payload.get(); + if (add.files.isEmpty()) { + payload.addError("No files specified"); + return; + } + PackageStoreAPI packageStoreAPI = coreContainer.getPackageStoreAPI(); + packageStoreAPI.validateFiles(add.files, true, s -> payload.addError(s)); + if (payload.hasError()) return; + Packages[] finalState = new Packages[1]; + try { + coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> { + Packages packages = null; + try { + packages = bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); + packages = packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + packages = new Packages(); + } + packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add)); + packages.znodeVersion = stat.getVersion() + 1; + finalState[0] = packages; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + finalState[0] = null; + handleZkErr(e); + } + if (finalState[0] != null) { +// succeeded in updating + pkgs = finalState[0]; + notifyAllNodesToSync(pkgs.znodeVersion); + packageLoader.refreshPackageConf(); + } + + } + + @Command(name = "delete") + public void del(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj payload) { + if (!checkEnabled(payload)) return; + Package.DelVersion delVersion = payload.get(); + try { + coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> { + Packages packages = null; + try { + packages = mapper.readValue(bytes, Packages.class); + packages = packages.copy(); + } catch (IOException e) { + packages = new Packages(); + } + + List versions = packages.packages.get(delVersion.pkg); + if (versions == null || versions.isEmpty()) { + payload.addError("No such package: " + delVersion.pkg); + return null;// no change + } + int idxToremove = -1; + for (int i = 0; i < versions.size(); i++) { + if (Objects.equals(versions.get(i).version, delVersion.version)) { + idxToremove = i; + break; + } + } + if (idxToremove == -1) { + payload.addError("No such version: " + delVersion.version); + return null; + } + versions.remove(idxToremove); + packages.znodeVersion = stat.getVersion() + 1; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + handleZkErr(e); + + } + + + } + + } + + private boolean checkEnabled(CommandOperation payload) { + if (!enablePackages) { + payload.addError("Package loading is not enabled , Start your nodes with -Denable.packages=true"); + return false; + } + return true; + } + + @EndPoint( + method = SolrRequest.METHOD.GET, + path = {"/cluster/package/", + "/cluster/package/{name}"}, + permission = PACKAGE_READ_PERM + ) + public class Read { + @Command() + public void get(SolrQueryRequest req, SolrQueryResponse rsp) { + String refresh = req.getParams().get("refreshPackage"); + if (refresh != null) { + packageLoader.notifyListeners(refresh); + return; + } + + int expectedVersion = req.getParams().getInt("expectedVersion", -1); + if (expectedVersion != -1) { + syncToVersion(expectedVersion); + } + String name = req.getPathTemplateValues().get("name"); + if (name == null) { + rsp.add("result", pkgs); + } else { + rsp.add("result", Collections.singletonMap(name, pkgs.packages.get(name))); + } + } + + private void syncToVersion(int expectedVersion) { + int origVersion = pkgs.znodeVersion; + for (int i = 0; i < 10; i++) { + log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion); + if (pkgs.znodeVersion >= expectedVersion) { + if (origVersion < pkgs.znodeVersion) { + packageLoader.refreshPackageConf(); + } + return; + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + } + try { + pkgs = readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + handleZkErr(e); + + } + + } + + } + + + } + + void notifyAllNodesToSync(int expected) { + for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) { + Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), + coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&expectedVersion" + expected, + Utils.JAVABINCONSUMER); + } + } + + public void handleZkErr(Exception e) { + log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); + } + + +} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java new file mode 100644 index 00000000000..0287f5e434d --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import java.lang.invoke.MethodHandles; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.SolrCore; +import org.apache.solr.logging.MDCLoggingContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PackageListeners { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String PACKAGE_VERSIONS = "PKG_VERSIONS"; + private SolrCore core; + + public PackageListeners(SolrCore core) { + this.core = core; + } + + // this registry only keeps a weak reference because it does not want to + // cause a memory leak if the listener forgets to unregister itself + private List> listeners = new ArrayList<>(); + + public synchronized void addListener(Listener listener) { + listeners.add(new SoftReference<>(listener)); + + } + + public synchronized void removeListener(Listener listener) { + Iterator> it = listeners.iterator(); + while (it.hasNext()) { + Reference ref = it.next(); + Listener pkgListener = ref.get(); + if (pkgListener == null || pkgListener == listener) { + it.remove(); + } + + } + + } + + synchronized void packagesUpdated(List pkgs) { + if(core != null) MDCLoggingContext.setCore(core); + try { + for (PackageLoader.Package pkgInfo : pkgs) { + invokeListeners(pkgInfo); + } + } finally { + if(core != null) MDCLoggingContext.clear(); + + } + } + + private synchronized void invokeListeners(PackageLoader.Package pkg) { + for (Reference ref : listeners) { + Listener listener = ref.get(); + if(listener == null) continue; + if (listener.packageName() == null || listener.packageName().equals(pkg.name())) { + listener.changed(pkg); + } + } + } + + public List getListeners() { + List result = new ArrayList<>(); + for (Reference ref : listeners) { + Listener l = ref.get(); + if (l != null) { + result.add(l); + } + } + return result; + } + + + public interface Listener { + /**Name of the package or null to loisten to all package changes + */ + String packageName(); + + PluginInfo pluginInfo(); + + void changed(PackageLoader.Package pkg); + + PackageLoader.Package.Version getPackageVersion(); + + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java new file mode 100644 index 00000000000..c86e4337d45 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.solr.common.MapWriter; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The class that holds a mapping of various packages and classloaders + */ +public class PackageLoader { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final CoreContainer coreContainer; + private final Map packageClassLoaders = new ConcurrentHashMap<>(); + + private PackageAPI.Packages myCopy; + + private PackageAPI packageAPI; + + + public PackageLoader(CoreContainer coreContainer) { + this.coreContainer = coreContainer; + packageAPI = new PackageAPI(coreContainer, this); + myCopy = packageAPI.pkgs; + + } + + public PackageAPI getPackageAPI() { + return packageAPI; + } + + public Package getPackage(String key) { + return packageClassLoaders.get(key); + } + + public Map getPackages() { + return Collections.EMPTY_MAP; + } + + public void refreshPackageConf() { + log.info("{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion); + + List updated = new ArrayList<>(); + Map> modified = getModified(myCopy, packageAPI.pkgs); + + for (Map.Entry> e : modified.entrySet()) { + if (e.getValue() != null) { + Package p = packageClassLoaders.get(e.getKey()); + if (e.getValue() != null && p == null) { + packageClassLoaders.put(e.getKey(), p = new Package(e.getKey())); + } + p.updateVersions(e.getValue()); + updated.add(p); + } else { + Package p = packageClassLoaders.remove(e.getKey()); + if (p != null) { + //other classes are holding to a reference to this objecec + // they should know that this is removed + p.markDeleted(); + } + } + } + for (SolrCore core : coreContainer.getCores()) { + core.getPackageListeners().packagesUpdated(updated); + } + } + + public Map> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) { + Map> changed = new HashMap<>(); + for (Map.Entry> e : newPkgs.packages.entrySet()) { + List versions = old.packages.get(e.getKey()); + if (versions != null) { + if (!Objects.equals(e.getValue(), versions)) { + log.info("Package {} is modified ", e.getKey()); + changed.put(e.getKey(), e.getValue()); + } + } else { + log.info("A new package: {} introduced", e.getKey()); + changed.put(e.getKey(), e.getValue()); + } + } + //some packages are deleted altogether + for (String s : old.packages.keySet()) { + if (!newPkgs.packages.keySet().contains(s)) { + log.info("Package: {} is removed althogether", s); + changed.put(s, null); + } + } + + return changed; + + } + + public void notifyListeners(String pkg) { + Package p = packageClassLoaders.get(pkg); + if(p != null){ + List l = Collections.singletonList(p); + for (SolrCore core : coreContainer.getCores()) { + core.getPackageListeners().packagesUpdated(l); + } + } + } + + /** + * represents a package definition in the packages.json + */ + public class Package { + final String name; + final Map myVersions = new ConcurrentHashMap<>(); + private List sortedVersions = new CopyOnWriteArrayList<>(); + String latest; + private boolean deleted; + + + Package(String name) { + this.name = name; + } + + public boolean isDeleted() { + return deleted; + } + + + private synchronized void updateVersions(List modified) { + for (PackageAPI.PkgVersion v : modified) { + Version version = myVersions.get(v.version); + if (version == null) { + log.info("A new version: {} added for package: {} with artifacts {}", v.version, this.name, v.files); + myVersions.put(v.version, new Version(this, v)); + sortedVersions.add(v.version); + } + } + + Set newVersions = new HashSet<>(); + for (PackageAPI.PkgVersion v : modified) { + newVersions.add(v.version); + } + for (String s : new HashSet<>(myVersions.keySet())) { + if (!newVersions.contains(s)) { + log.info("version: {} is removed from package: {}", s, this.name); + sortedVersions.remove(s); + myVersions.remove(s); + } + } + + sortedVersions.sort(String::compareTo); + if (sortedVersions.size() > 0) { + String latest = sortedVersions.get(sortedVersions.size() - 1); + if (!latest.equals(this.latest)) { + log.info("version: {} is the new latest in package: {}", latest, this.name); + } + this.latest = latest; + } else { + log.error("latest version: null"); + latest = null; + } + + } + + + public Version getLatest() { + return latest == null ? null : myVersions.get(latest); + } + + public Version getLatest(String lessThan) { + if (lessThan == null) { + return getLatest(); + } + String latest = findBiggest(lessThan, new ArrayList(sortedVersions)); + return latest == null ? null : myVersions.get(latest); + } + + public String name() { + return name; + } + + private void markDeleted() { + deleted = true; + myVersions.clear(); + sortedVersions.clear(); + latest = null; + + } + + public class Version implements MapWriter { + private final Package parent; + private SolrResourceLoader loader; + + private final PackageAPI.PkgVersion version; + + @Override + public void writeMap(EntryWriter ew) throws IOException { + ew.put("package", parent.name()); + version.writeMap(ew); + } + + Version(Package parent, PackageAPI.PkgVersion v) { + this.parent = parent; + this.version = v; + List paths = new ArrayList<>(); + for (String file : version.files) { + paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file)); + } + + try { + loader = new SolrResourceLoader( + "PACKAGE_LOADER: " + parent.name() + ":" + version, + paths, + coreContainer.getResourceLoader().getInstancePath(), + coreContainer.getResourceLoader().getClassLoader()); + } catch (MalformedURLException e) { + log.error("Could not load classloader ", e); + } + } + + public String getVersion() { + return version.version; + } + + public Collection getFiles() { + return Collections.unmodifiableList(version.files); + } + + public SolrResourceLoader getLoader() { + return loader; + } + } + } + + private static String findBiggest(String lessThan, List sortedList) { + String latest = null; + for (String v : sortedList) { + if (v.compareTo(lessThan) < 1) { + latest = v; + } else break; + } + return latest; + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java new file mode 100644 index 00000000000..63facdec0d3 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import java.lang.invoke.MethodHandles; + +import org.apache.solr.core.PluginBag; +import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.RequestParams; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrCore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PackagePluginHolder extends PluginBag.PluginHolder { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String LATEST = "$LATEST"; + + private final SolrCore core; + private final SolrConfig.SolrPluginInfo pluginMeta; + private PackageLoader.Package.Version pkgVersion; + private PluginInfo info; + + + public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) { + super(info); + this.core = core; + this.pluginMeta = pluginMeta; + this.info = info; + + reload(core.getCoreContainer().getPackageLoader().getPackage(info.pkgName)); + core.getPackageListeners().addListener(new PackageListeners.Listener() { + @Override + public String packageName() { + return info.pkgName; + } + + @Override + public PluginInfo pluginInfo() { + return info; + } + + @Override + public void changed(PackageLoader.Package pkg) { + reload(pkg); + + } + + @Override + public PackageLoader.Package.Version getPackageVersion() { + return pkgVersion; + } + + }); + } + + private String maxVersion() { + RequestParams.ParamSet p = core.getSolrConfig().getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS); + if (p == null) { + return null; + } + Object o = p.get().get(info.pkgName); + if (o == null || LATEST.equals(o)) return null; + return o.toString(); + } + + + private synchronized void reload(PackageLoader.Package pkg) { + String lessThan = maxVersion(); + PackageLoader.Package.Version newest = pkg.getLatest(lessThan); + if (newest == null) { + log.error("No latest version available for package : {}", pkg.name()); + return; + } + if (lessThan != null) { + PackageLoader.Package.Version pkgLatest = pkg.getLatest(); + if (pkgLatest != newest) { + log.info("Using version :{}. latest is {}, params.json has config {} : {}", newest.getVersion(), pkgLatest.getVersion(), pkg.name(), lessThan); + } + } + + if (pkgVersion != null) { + if (newest == pkgVersion) { + //I'm already using the latest classloder in the package. nothing to do + return; + } + } + + log.info("loading plugin: {} -> {} using package {}:{}", + pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion()); + + Object instance = SolrCore.createInstance(pluginInfo.className, + pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader()); + PluginBag.initInstance(instance, pluginInfo); + T old = inst; + inst = (T) instance; + pkgVersion = newest; + if (old instanceof AutoCloseable) { + AutoCloseable closeable = (AutoCloseable) old; + try { + closeable.close(); + } catch (Exception e) { + log.error("error closing plugin", e); + } + } + } + +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java index a4c7c0d0730..b5d409967ec 100644 --- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java +++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java @@ -53,6 +53,9 @@ public interface PermissionNameProvider { METRICS_HISTORY_READ_PERM("metrics-history-read", null), FILESTORE_READ_PERM("filestore-read", null), FILESTORE_WRITE_PERM("filestore-write", null), + PACKAGE_EDIT_PERM("package-edit", null), + PACKAGE_READ_PERM("package-read", null), + ALL("all", unmodifiableSet(new HashSet<>(asList("*", null)))) ; final String name; diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java index eb3c08b2169..5ddaf8b1013 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java @@ -23,6 +23,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import com.google.common.collect.ImmutableMap; import org.apache.solr.common.SolrException; @@ -33,9 +34,12 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.core.PluginBag; import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; +import org.apache.solr.pkg.PackagePluginHolder; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory; import org.apache.solr.util.plugin.PluginInfoInitialized; import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; @@ -126,8 +130,7 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized // wrap in an ArrayList so we know we know we can do fast index lookups // and that add(int,Object) is supported - List list = new ArrayList<> - (solrCore.initPlugins(info.getChildren("processor"),UpdateRequestProcessorFactory.class,null)); + List list = createProcessors(info); if(list.isEmpty()){ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, @@ -170,6 +173,23 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized } + private List createProcessors(PluginInfo info) { + List processors = info.getChildren("processor"); + return processors.stream().map(it -> { + if(it.pkgName == null){ + return solrCore.createInitInstance(it, UpdateRequestProcessorFactory.class, + UpdateRequestProcessorFactory.class.getSimpleName(), null); + + } else { + return new LazyUpdateRequestProcessorFactory(new PackagePluginHolder( + it, + solrCore, + SolrConfig.classVsSolrPluginInfo.get(UpdateRequestProcessorFactory.class.getName()))); + } + }).collect(Collectors.toList()); + } + + /** * Creates a chain backed directly by the specified list. Modifications to * the array will affect future calls to createProcessor @@ -328,7 +348,7 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized public static class LazyUpdateProcessorFactoryHolder extends PluginBag.PluginHolder { private volatile UpdateRequestProcessorFactory lazyFactory; - public LazyUpdateProcessorFactoryHolder(final PluginBag.LazyPluginHolder holder) { + public LazyUpdateProcessorFactoryHolder(final PluginBag.PluginHolder holder) { super(holder.getPluginInfo()); lazyFactory = new LazyUpdateRequestProcessorFactory(holder); } @@ -339,27 +359,20 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized return lazyFactory; } - public class LazyUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory { - private final PluginBag.LazyPluginHolder holder; - UpdateRequestProcessorFactory delegate; + public static class LazyUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory { + private final PluginBag.PluginHolder holder; - public LazyUpdateRequestProcessorFactory(PluginBag.LazyPluginHolder holder) { + public LazyUpdateRequestProcessorFactory(PluginBag.PluginHolder holder) { this.holder = holder; } public UpdateRequestProcessorFactory getDelegate() { - return delegate; + return holder.get(); } @Override public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) { - if (delegate != null) return delegate.getInstance(req, rsp, next); - - synchronized (this) { - if (delegate == null) - delegate = (UpdateRequestProcessorFactory) holder.get(); - } - return delegate.getInstance(req, rsp, next); + return holder.get().getInstance(req, rsp, next); } } } diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java new file mode 100644 index 00000000000..00deb6315b8 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -0,0 +1,495 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.Callable; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.request.beans.Package; +import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.solr.cloud.ConfigRequest; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.MapWriterMap; +import org.apache.solr.common.NavigableObject; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.Utils; +import org.apache.solr.filestore.TestDistribPackageStore; +import org.apache.solr.util.LogLevel; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.Stat; +import org.junit.Test; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; +import static org.apache.solr.common.params.CommonParams.JAVABIN; +import static org.apache.solr.common.params.CommonParams.WT; +import static org.apache.solr.core.TestDynamicLoading.getFileContent; +import static org.apache.solr.filestore.TestDistribPackageStore.readFile; + +@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") +public class TestPackages extends SolrCloudTestCase { + + @Test + public void testPluginLoading() throws Exception { + System.setProperty("enable.packages", "true"); + MiniSolrCloudCluster cluster = + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + try { + String FILE1 = "/mypkg/runtimelibs.jar"; + String FILE2 = "/mypkg/runtimelibs_v2.jar"; + String FILE3 = "/mypkg/runtimelibs_v3.jar"; + String COLLECTION_NAME = "testPluginLoadingColl"; + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + cluster.getZkClient().makePath("/keys/exe", true); + cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); + postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, + "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + + Package.AddVersion add = new Package.AddVersion(); + add.version = "1.0"; + add.pkg = "mypkg"; + add.files = Arrays.asList(new String[]{FILE1}); + V2Request req = new V2Request.Builder("/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + + req.process(cluster.getSolrClient()); + + + CollectionAdminRequest + .createCollection(COLLECTION_NAME, "conf", 2, 2) + .setMaxShardsPerNode(100) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); + + TestDistribPackageStore.assertResponseValues(10, + () -> new V2Request.Builder("/cluster/package"). + withMethod(SolrRequest.METHOD.GET) + .build().process(cluster.getSolrClient()), + Utils.makeMap( + ":result:packages:mypkg[0]:version", "1.0", + ":result:packages:mypkg[0]:files[0]", FILE1 + )); + + String payload = "{\n" + + "'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler' }," + + "'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent' }," + + "'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter' }" + + "}"; + cluster.getSolrClient().request(new ConfigRequest(payload) { + @Override + public String getCollection() { + return COLLECTION_NAME; + } + }); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "queryResponseWriter", "json1", + "mypkg", "1.0" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "searchComponent", "get", + "mypkg", "1.0" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "requestHandler", "/runtime", + "mypkg", "1.0" ); + + + + executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()), + Utils.JAVABINCONSUMER, + Utils.makeMap("class", "org.apache.solr.core.RuntimeLibReqHandler")); + + executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()), + Utils.JSONCONSUMER, + Utils.makeMap("class", "org.apache.solr.core.RuntimeLibSearchComponent", + "Version","1")); + + + executeReq( "/" + COLLECTION_NAME + "/runtime?wt=json1", cluster.getRandomJetty(random()), + Utils.JSONCONSUMER, + Utils.makeMap("wt", "org.apache.solr.core.RuntimeLibResponseWriter")); + + //now upload the second jar + postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2, + "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); + + //add the version using package API + add.version = "1.1"; + add.files = Arrays.asList(new String[]{FILE2}); + req.process(cluster.getSolrClient()); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "queryResponseWriter", "json1", + "mypkg", "1.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "searchComponent", "get", + "mypkg", "1.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "requestHandler", "/runtime", + "mypkg", "1.1" ); + + executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()), + Utils.JSONCONSUMER, + Utils.makeMap( "Version","2")); + + + //now upload the third jar + postFileAndWait(cluster, "runtimecode/runtimelibs_v3.jar.bin", FILE3, + "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q=="); + + add.version = "2.1"; + add.files = Arrays.asList(new String[]{FILE3}); + req.process(cluster.getSolrClient()); + + //now let's verify that the classes are updated + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "queryResponseWriter", "json1", + "mypkg", "2.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "searchComponent", "get", + "mypkg", "2.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "requestHandler", "/runtime", + "mypkg", "2.1" ); + + executeReq( "/" + COLLECTION_NAME + "/runtime?wt=json", cluster.getRandomJetty(random()), + Utils.JSONCONSUMER, + Utils.makeMap("Version","2")); + + + Package.DelVersion delVersion = new Package.DelVersion(); + delVersion.pkg = "mypkg"; + delVersion.version = "1.0"; + V2Request delete = new V2Request.Builder("/cluster/package") + .withMethod(SolrRequest.METHOD.POST) + .forceV2(true) + .withPayload(Collections.singletonMap("delete", delVersion)) + .build(); + delete.process(cluster.getSolrClient()); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "queryResponseWriter", "json1", + "mypkg", "2.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "searchComponent", "get", + "mypkg", "2.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "requestHandler", "/runtime", + "mypkg", "2.1" ); + + // now remove the hughest version. So, it will roll back to the next highest one + delVersion.version = "2.1"; + delete.process(cluster.getSolrClient()); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "queryResponseWriter", "json1", + "mypkg", "1.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "searchComponent", "get", + "mypkg", "1.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "requestHandler", "/runtime", + "mypkg", "1.1" ); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add("collection", COLLECTION_NAME); + new GenericSolrRequest(SolrRequest.METHOD.POST, "/config/params", params ){ + @Override + public RequestWriter.ContentWriter getContentWriter(String expectedType) { + return new RequestWriter.StringPayloadContentWriter("{set:{PKG_VERSIONS:{mypkg : '1.1'}}}", + ClientUtils.TEXT_JSON); + } + }.process(cluster.getSolrClient()) ; + + add.version = "2.1"; + add.files = Arrays.asList(new String[]{FILE3}); + req.process(cluster.getSolrClient()); + + //the collections mypkg is set to use version 1.1 + //so no upgrade + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "queryResponseWriter", "json1", + "mypkg", "1.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "searchComponent", "get", + "mypkg", "1.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "requestHandler", "/runtime", + "mypkg", "1.1" ); + + new GenericSolrRequest(SolrRequest.METHOD.POST, "/config/params", params ){ + @Override + public RequestWriter.ContentWriter getContentWriter(String expectedType) { + return new RequestWriter.StringPayloadContentWriter("{set:{PKG_VERSIONS:{mypkg : '2.1'}}}", + ClientUtils.TEXT_JSON); + } + }.process(cluster.getSolrClient()) ; + + //now, let's force every collection using 'mypkg' to refresh + //so that it uses version 2.1 + new V2Request.Builder("/cluster/package") + .withMethod(SolrRequest.METHOD.POST) + .withPayload("{refresh : mypkg}") + .forceV2(true) + .build() + .process(cluster.getSolrClient()); + + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "queryResponseWriter", "json1", + "mypkg", "2.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "searchComponent", "get", + "mypkg", "2.1" ); + + verifyCmponent(cluster.getSolrClient(), + COLLECTION_NAME, "requestHandler", "/runtime", + "mypkg", "2.1" ); + + } finally { + cluster.shutdown(); + } + + } + + private void executeReq(String uri, JettySolrRunner jetty, Utils.InputStreamConsumer parser, Map expected) throws Exception { + try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()){ + TestDistribPackageStore.assertResponseValues(10, + () -> { + Object o = Utils.executeGET(client.getHttpClient(), + jetty.getBaseUrl() + uri, parser); + if(o instanceof NavigableObject) return (NavigableObject) o; + if(o instanceof Map) return new MapWriterMap((Map) o); + throw new RuntimeException("Unknown response"); + }, expected); + + } + } + + private void verifyCmponent(SolrClient client, String COLLECTION_NAME, + String componentType, String componentName, String pkg, String version) throws Exception { + SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, + WT, JAVABIN, + "componentName", componentName, + "meta", "true")); + + String s = "queryResponseWriter"; + GenericSolrRequest req1 = new GenericSolrRequest(SolrRequest.METHOD.GET, + "/config/" + componentType, params); + TestDistribPackageStore.assertResponseValues(10, + client, + req1, Utils.makeMap( + ":config:" + componentType + ":" + componentName + ":_packageinfo_:package", pkg, + ":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version + )); + } + + @Test + public void testAPI() throws Exception { + System.setProperty("enable.packages", "true"); + MiniSolrCloudCluster cluster = + configureCluster(4) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + try { + String errPath = "/error/details[0]/errorMessages[0]"; + String FILE1 = "/mypkg/v.0.12/jar_a.jar"; + String FILE2 = "/mypkg/v.0.12/jar_b.jar"; + String FILE3 = "/mypkg/v.0.13/jar_a.jar"; + + Package.AddVersion add = new Package.AddVersion(); + add.version = "0.12"; + add.pkg = "test_pkg"; + add.files = Arrays.asList(new String[]{FILE1, FILE2}); + V2Request req = new V2Request.Builder("/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("add", add)) + .build(); + + //the files is not yet there. The command should fail with error saying "No such file" + expectError(req, cluster.getSolrClient(), errPath, "No such file :"); + + + //post the jar file. No signature is sent + postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null); + + + add.files = Arrays.asList(new String[]{FILE1}); + expectError(req, cluster.getSolrClient(), errPath, + FILE1 + " has no signature"); + //now we upload the keys + byte[] derFile = readFile("cryptokeys/pub_key512.der"); + cluster.getZkClient().makePath("/keys/exe", true); + cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); + //and upload the same file with a different name but it has proper signature + postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE2, + "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + // with correct signature + //after uploading the file, let's delete the keys to see if we get proper error message + cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true); + add.files = Arrays.asList(new String[]{FILE2}); + expectError(req, cluster.getSolrClient(), errPath, + "ZooKeeper does not have any public keys"); + + //Now lets' put the keys back + cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); + + //this time we have a file with proper signature, public keys are in ZK + // so the add {} command should succeed + req.process(cluster.getSolrClient()); + + //Now verify the data in ZK + TestDistribPackageStore.assertResponseValues(1, + () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, + null, new Stat(), true))), + Utils.makeMap( + ":packages:test_pkg[0]:version", "0.12", + ":packages:test_pkg[0]:files[0]", FILE1 + )); + + //post a new jar with a proper signature + postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE3, + "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA=="); + + + //this time we are adding the second version of the package (0.13) + add.version = "0.13"; + add.pkg = "test_pkg"; + add.files = Arrays.asList(new String[]{FILE3}); + + //this request should succeed + req.process(cluster.getSolrClient()); + //no verify the data (/packages.json) in ZK + TestDistribPackageStore.assertResponseValues(1, + () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, + null, new Stat(), true))), + Utils.makeMap( + ":packages:test_pkg[1]:version", "0.13", + ":packages:test_pkg[1]:files[0]", FILE3 + )); + + //Now we will just delete one version + Package.DelVersion delVersion = new Package.DelVersion(); + delVersion.version = "0.1";//this version does not exist + delVersion.pkg = "test_pkg"; + req = new V2Request.Builder("/cluster/package") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(Collections.singletonMap("delete", delVersion)) + .build(); + + //we are expecting an error + expectError(req, cluster.getSolrClient(), errPath, "No such version:"); + + delVersion.version = "0.12";//correct version. Should succeed + req.process(cluster.getSolrClient()); + //Verify with ZK that the data is correcy + TestDistribPackageStore.assertResponseValues(1, + () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH, + null, new Stat(), true))), + Utils.makeMap( + ":packages:test_pkg[0]:version", "0.13", + ":packages:test_pkg[0]:files[0]", FILE2 + )); + + + //So far we have been verifying the details with ZK directly + //use the package read API to verify with each node that it has the correct data + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + String path = jetty.getBaseUrl().toString().replace("/solr", "/api") + "/cluster/package?wt=javabin"; + TestDistribPackageStore.assertResponseValues(10, new Callable() { + @Override + public NavigableObject call() throws Exception { + try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) { + return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), path, Utils.JAVABINCONSUMER); + } + } + }, Utils.makeMap( + ":result:packages:test_pkg[0]:version", "0.13", + ":result:packages:test_pkg[0]:files[0]", FILE3 + )); + } + } finally { + cluster.shutdown(); + } + } + + static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { + ByteBuffer fileContent = getFileContent(fname); + String sha512 = DigestUtils.sha512Hex(fileContent.array()); + + TestDistribPackageStore.postFile(cluster.getSolrClient(), + fileContent, + path, sig);// has file, but no signature + + TestDistribPackageStore.waitForAllNodesHaveFile(cluster, path, Utils.makeMap( + ":files:" + path + ":sha512", + sha512 + ), false); + } + + private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException { + try { + req.process(client); + fail("should have failed with message : " + expectErrorMsg); + } catch (BaseHttpSolrClient.RemoteExecutionException e) { + String msg = e.getMetaData()._getStr(errPath, ""); + assertTrue("should have failed with message: " + expectErrorMsg + "actual message : " + msg, + msg.contains(expectErrorMsg) + ); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java b/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java index 889b0bf5786..6cee3d9c1d8 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java +++ b/solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java @@ -31,7 +31,7 @@ public class RuntimeUrp extends SimpleUpdateProcessorFactory { List names = new ArrayList<>(); for (UpdateRequestProcessorFactory p : processorChain.getProcessors()) { if (p instanceof UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) { - p = ((UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) p).delegate; + p = ((UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) p).getDelegate(); } names.add(p.getClass().getSimpleName()); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java new file mode 100644 index 00000000000..62bf6ecf1f1 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.request.beans; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.solr.common.util.ReflectMapWriter; + +/**Just a container class for POJOs used in Package APIs + * + */ +public class Package { + public static class AddVersion implements ReflectMapWriter { + @JsonProperty(value = "package", required = true) + public String pkg; + @JsonProperty(required = true) + public String version; + @JsonProperty(required = true) + public List files; + + } + + public static class DelVersion implements ReflectMapWriter { + @JsonProperty(value = "package", required = true) + public String pkg; + @JsonProperty(required = true) + public String version; + + } +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/pakage-info.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/pakage-info.java new file mode 100644 index 00000000000..b419ef87aeb --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/pakage-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Data objects used in V2 Requests with jackson bindings + */ +package org.apache.solr.client.solrj.request.beans; + + diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java index 98f0d9ad0bd..b8105ab098f 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java @@ -34,6 +34,7 @@ import java.nio.file.Path; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -353,6 +354,10 @@ public class SolrZkClient implements Closeable { } public void atomicUpdate(String path, Function editor) throws KeeperException, InterruptedException { + atomicUpdate(path, (stat, bytes) -> editor.apply(bytes)); + } + + public void atomicUpdate(String path, BiFunction editor) throws KeeperException, InterruptedException { for (; ; ) { byte[] modified = null; byte[] zkData = null; @@ -360,7 +365,7 @@ public class SolrZkClient implements Closeable { try { if (exists(path, true)) { zkData = getData(path, null, s, true); - modified = editor.apply(zkData); + modified = editor.apply(s, zkData); if (modified == null) { //no change , no need to persist return; @@ -368,7 +373,7 @@ public class SolrZkClient implements Closeable { setData(path, modified, s.getVersion(), true); break; } else { - modified = editor.apply(null); + modified = editor.apply(s,null); if (modified == null) { //no change , no need to persist return; diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index dcf7d9e31d2..5dea5b0b360 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -118,6 +118,7 @@ public class ZkStateReader implements SolrCloseable { public static final String SOLR_AUTOSCALING_TRIGGER_STATE_PATH = "/autoscaling/triggerState"; public static final String SOLR_AUTOSCALING_NODE_ADDED_PATH = "/autoscaling/nodeAdded"; public static final String SOLR_AUTOSCALING_NODE_LOST_PATH = "/autoscaling/nodeLost"; + public static final String SOLR_PKGS_PATH = "/packages.json"; public static final String DEFAULT_SHARD_PREFERENCES = "defaultShardPreferences"; public static final String REPLICATION_FACTOR = "replicationFactor"; diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ReflectMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/util/ReflectMapWriter.java index 0193aeaddb0..3a788917dfb 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/ReflectMapWriter.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/ReflectMapWriter.java @@ -23,7 +23,7 @@ import java.lang.reflect.Modifier; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.solr.common.MapWriter; - +// An implementation of MapWriter which is annotated with Jackson annotations public interface ReflectMapWriter extends MapWriter { @Override