Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-13822: Isolated Classloading from packages (#957) 

SOLR-13822: A Package management system with the following features. A packages.json in ZK to store
  the configuration, APIs to read/edit them and isolated classloaders to load the classes from
  those packages if the 'class' attribute is prefixed with `<package-name>:`
This commit is contained in:
Noble Paul 2019-10-24 08:55:11 +11:00 committed by GitHub
parent 3ae8204248
commit 98f08d39aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1868 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
solr/CHANGES.txt
View File

@ -100,7 +100,11 @@ Upgrade Notes
New Features
---------------------
(No changes)
* SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya)
* SOLR-13822: A Package management system with the following features. A packages.json in ZK to store
the configuration, APIs to read/edit them and isolated classloaders to load the classes from
hose packages if the 'class' attribute is prefixed with `<package-name>:` (noble, Ishan Chattopadhyaya)
Improvements
---------------------
@ -194,8 +198,6 @@ New Features
* SOLR-8241: Add CaffeineCache, an efficient implementation of SolrCache.(Ben Manes, Shawn Heisey, David Smiley, Andrzej Bialecki)
* SOLR-13821: A Package store to store and load package artefacts (noble, Ishan Chattopadhyaya)
* SOLR-13298: Allow zplot to plot matrices (Joel Bernstein)
Improvements

4
solr/core/src/java/org/apache/solr/api/AnnotatedApi.java
View File

@ -227,6 +227,7 @@ public class AnnotatedApi extends Api implements PermissionNameProvider {
}
if (isWrappedInPayloadObj) {
PayloadObj<Object> payloadObj = new PayloadObj<>(cmd.name, cmd.getCommandData(), o);
cmd = payloadObj;
method.invoke(obj, req, rsp, payloadObj);
} else {
method.invoke(obj, req, rsp, o);
@ -239,10 +240,13 @@ public class AnnotatedApi extends Api implements PermissionNameProvider {
} catch (SolrException se) {
log.error("Error executing command ", se);
throw se;
} catch (InvocationTargetException ite) {
log.error("Error executing command ", ite);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ite.getCause());
} catch (Exception e) {
log.error("Error executing command : ", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}

28
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -101,6 +101,7 @@ import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.search.SolrFieldCacheBean;
@ -224,6 +225,7 @@ public class CoreContainer {
protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler;
private PackageStoreAPI packageStoreAPI;
private PackageLoader packageLoader;
// Bits for the state variable.
@ -582,6 +584,10 @@ public class CoreContainer {
return replayUpdatesExecutor;
}
public PackageLoader getPackageLoader() {
return packageLoader;
}
public PackageStoreAPI getPackageStoreAPI() {
return packageStoreAPI;
}
@ -736,8 +742,12 @@ public class CoreContainer {
if (isZooKeeperAware()) {
metricManager.loadClusterReporters(metricReporters, this);
packageLoader = new PackageLoader(this);
containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP);
containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP);
}
// setup executor to load cores in parallel
ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(
ExecutorUtil.newMDCAwareFixedThreadPool(
@ -1210,15 +1220,15 @@ public class CoreContainer {
* that calls solrCores.waitAddPendingCoreOps(...) and solrCores.removeFromPendingOps(...)
*
* <pre>
* <code>
* try {
* solrCores.waitAddPendingCoreOps(dcore.getName());
* createFromDescriptor(...);
* } finally {
* solrCores.removeFromPendingOps(dcore.getName());
* }
* </code>
* </pre>
* <code>
* try {
* solrCores.waitAddPendingCoreOps(dcore.getName());
* createFromDescriptor(...);
* } finally {
* solrCores.removeFromPendingOps(dcore.getName());
* }
* </code>
* </pre>
* <p>
* Trying to put the waitAddPending... in this method results in Bad Things Happening due to race conditions.
* getCore() depends on getting the core returned _if_ it's in the pending list due to some other thread opening it.

28
solr/core/src/java/org/apache/solr/core/PluginBag.java
View File

@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@ -43,8 +44,10 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.util.CryptoKeys;
import org.apache.solr.util.SimplePostTool;
@ -97,7 +100,7 @@ public class PluginBag<T> implements AutoCloseable {
this(klass, core, false);
}
static void initInstance(Object inst, PluginInfo info) {
public static void initInstance(Object inst, PluginInfo info) {
if (inst instanceof PluginInfoInitialized) {
((PluginInfoInitialized) inst).init(info);
} else if (inst instanceof NamedListInitializedPlugin) {
@ -138,14 +141,23 @@ public class PluginBag<T> implements AutoCloseable {
log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name);
return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader(), false);
} else {
T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader());
initInstance(inst, info);
return new PluginHolder<>(info, inst);
if (info.pkgName != null) {
PackagePluginHolder<T> holder = new PackagePluginHolder<>(info, core, meta);
return meta.clazz == UpdateRequestProcessorFactory.class ?
new PluginHolder(info, new LazyUpdateProcessorFactoryHolder(holder)) :
holder;
} else {
T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader(info.pkgName));
initInstance(inst, info);
return new PluginHolder<>(info, inst);
}
}
}
/** make a plugin available in an alternate name. This is an internal API and not for public use
* @param src key in which the plugin is already registered
/**
* make a plugin available in an alternate name. This is an internal API and not for public use
*
* @param src key in which the plugin is already registered
* @param target the new key in which the plugin should be aliased to. If target exists already, the alias fails
* @return flag if the operation is successful or not
*/
@ -340,8 +352,8 @@ public class PluginBag<T> implements AutoCloseable {
* An indirect reference to a plugin. It just wraps a plugin instance.
* subclasses may choose to lazily load the plugin
*/
public static class PluginHolder<T> implements AutoCloseable {
private T inst;
public static class PluginHolder<T> implements Supplier<T>, AutoCloseable {
protected T inst;
protected final PluginInfo pluginInfo;
boolean registerAPI = false;

44
solr/core/src/java/org/apache/solr/core/PluginInfo.java
View File

@ -16,14 +16,20 @@
*/
package org.apache.solr.core;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Pair;
import org.apache.solr.util.DOMUtil;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.util.*;
import static java.util.Arrays.asList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableMap;
@ -35,27 +41,51 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME;
*
*/
public class PluginInfo implements MapSerializable {
public final String name, className, type;
public final String name, className, type, pkgName;
public final NamedList initArgs;
public final Map<String, String> attributes;
public final List<PluginInfo> children;
private boolean isFromSolrConfig;
public PluginInfo(String type, Map<String, String> attrs, NamedList initArgs, List<PluginInfo> children) {
this.type = type;
this.name = attrs.get(NAME);
this.className = attrs.get(CLASS_NAME);
Pair<String, String> parsed = parseClassName(attrs.get(CLASS_NAME));
this.className = parsed.second();
this.pkgName = parsed.first();
this.initArgs = initArgs;
attributes = unmodifiableMap(attrs);
this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children);
isFromSolrConfig = false;
}
/** class names can be prefixed with package name e.g: my_package:my.pkg.Class
* This checks if it is a package name prefixed classname.
* the return value has first = package name & second = class name
*/
static Pair<String,String > parseClassName(String name) {
String pkgName = null;
String className = name;
if (name != null) {
int colonIdx = name.indexOf(':');
if (colonIdx > -1) {
pkgName = name.substring(0, colonIdx);
className = name.substring(colonIdx + 1);
}
}
return new Pair<>(pkgName, className);
}
public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) {
type = node.getNodeName();
name = DOMUtil.getAttr(node, NAME, requireName ? err : null);
className = DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null);
Pair<String, String> parsed = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null));
className = parsed.second();
pkgName = parsed.first();
initArgs = DOMUtil.childNodesToNamedList(node);
attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes()));
children = loadSubPlugins(node);
@ -85,7 +115,9 @@ public class PluginInfo implements MapSerializable {
}
this.type = type;
this.name = (String) m.get(NAME);
this.className = (String) m.get(CLASS_NAME);
Pair<String, String> parsed = parseClassName((String) m.get(CLASS_NAME));
this.className = parsed.second();
this.pkgName = parsed.first();
attributes = unmodifiableMap(m);
this.children = Collections.<PluginInfo>emptyList();
isFromSolrConfig = true;

9
solr/core/src/java/org/apache/solr/core/RequestParams.java
View File

@ -250,9 +250,18 @@ public class RequestParams implements MapSerializable {
return m1;
}
/**
* @param type one of defaults, appends, invariants
*/
public VersionedParams getParams(String type) {
return paramsMap.get(type);
}
/**get the raw map
*/
public Map<String, Object> get() {
return defaults;
}
}
public static class VersionedParams extends MapSolrParams {

22
solr/core/src/java/org/apache/solr/core/SolrCore.java
View File

@ -110,6 +110,8 @@ import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.BinaryResponseWriter;
@ -238,6 +240,8 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
public volatile boolean indexEnabled = true;
public volatile boolean readOnly = false;
private PackageListeners packageListeners = new PackageListeners(this);
public Set<String> getMetricNames() {
return metricNames;
}
@ -262,6 +266,10 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return restManager;
}
public PackageListeners getPackageListeners() {
return packageListeners;
}
static int boolean_query_max_clause_count = Integer.MIN_VALUE;
private ExecutorService coreAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Async Task");
@ -275,6 +283,18 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return resourceLoader;
}
/** Gets the SolrResourceLoader for a given package
* @param pkg The package name
*/
public SolrResourceLoader getResourceLoader(String pkg) {
if (pkg == null) {
return resourceLoader;
}
PackageLoader.Package aPackage = coreContainer.getPackageLoader().getPackage(pkg);
PackageLoader.Package.Version latest = aPackage.getLatest();
return latest.getLoader();
}
/**
* Gets the configuration resource name used by this core instance.
*
@ -857,7 +877,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
public <T extends Object> T createInitInstance(PluginInfo info, Class<T> cast, String msg, String defClassName) {
if (info == null) return null;
T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader());
T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName));
if (o instanceof PluginInfoInitialized) {
((PluginInfoInitialized) o).init(info);
} else if (o instanceof NamedListInitializedPlugin) {

334
solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
View File

@ -28,6 +28,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.CharacterCodingException;
@ -81,9 +82,8 @@ import org.slf4j.LoggerFactory;
/**
* @since solr 1.3
*/
public class SolrResourceLoader implements ResourceLoader,Closeable
{
*/
public class SolrResourceLoader implements ResourceLoader, Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
static final String project = "solr";
@ -96,31 +96,33 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
};
private static final java.lang.String SOLR_CORE_NAME = "solr.core.name";
private static Set<String> loggedOnce = new ConcurrentSkipListSet<>();
private static final Charset UTF_8 = StandardCharsets.UTF_8;
private String name = "";
protected URLClassLoader classLoader;
private final Path instanceDir;
private String dataDir;
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
private static final Charset UTF_8 = StandardCharsets.UTF_8;
private final Properties coreProperties;
private volatile boolean live;
// Provide a registry so that managed resources can register themselves while the XML configuration
// documents are being parsed ... after all are registered, they are asked by the RestManager to
// initialize themselves. This two-step process is required because not all resources are available
// (such as the SolrZkClient) when XML docs are being parsed.
private RestManager.Registry managedResourceRegistry;
public synchronized RestManager.Registry getManagedResourceRegistry() {
if (managedResourceRegistry == null) {
managedResourceRegistry = new RestManager.Registry();
managedResourceRegistry = new RestManager.Registry();
}
return managedResourceRegistry;
return managedResourceRegistry;
}
public SolrResourceLoader() {
@ -134,11 +136,20 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
* found in the "lib/" directory in the specified instance directory.
* If the instance directory is not specified (=null), SolrResourceLoader#locateInstanceDir will provide one.
*/
public SolrResourceLoader(Path instanceDir, ClassLoader parent)
{
public SolrResourceLoader(Path instanceDir, ClassLoader parent) {
this(instanceDir, parent, null);
}
public SolrResourceLoader(String name, List<Path> classpath, Path instanceDir, ClassLoader parent) throws MalformedURLException {
this(instanceDir, parent);
this.name = name;
for (Path path : classpath) {
addToClassLoader(path.toUri().normalize().toURL());
}
}
public SolrResourceLoader(Path instanceDir) {
this(instanceDir, null, null);
}
@ -157,7 +168,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
if (instanceDir == null) {
this.instanceDir = SolrResourceLoader.locateSolrHome().toAbsolutePath().normalize();
log.debug("new SolrResourceLoader for deduced Solr Home: '{}'", this.instanceDir);
} else{
} else {
this.instanceDir = instanceDir.toAbsolutePath().normalize();
log.debug("new SolrResourceLoader for directory: '{}'", this.instanceDir);
}
@ -167,7 +178,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
this.classLoader = URLClassLoader.newInstance(new URL[0], parent);
/*
/*
* Skip the lib subdirectory when we are loading from the solr home.
* Otherwise load it, so core lib directories still get loaded.
* The default sharedLib will pick this up later, and if the user has
@ -264,6 +275,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Utility method to get the URLs of all paths under a given directory that match a filter
*
* @param libDir the root directory
* @param filter the filter
* @return all matching URLs
@ -296,8 +308,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Utility method to get the URLs of all paths under a given directory that match a regex
*
* @param libDir the root directory
* @param regex the regex as a String
* @param regex the regex as a String
* @return all matching URLs
* @throws IOException on error
*/
@ -310,15 +323,17 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
});
}
/** Ensures a directory name always ends with a '/'. */
/**
* Ensures a directory name always ends with a '/'.
*/
public static String normalizeDir(String path) {
return ( path != null && (!(path.endsWith("/") || path.endsWith("\\"))) )? path + File.separator : path;
return (path != null && (!(path.endsWith("/") || path.endsWith("\\")))) ? path + File.separator : path;
}
public String[] listConfigDir() {
File configdir = new File(getConfigDir());
if( configdir.exists() && configdir.isDirectory() ) {
if (configdir.exists() && configdir.isDirectory()) {
return configdir.list();
} else {
return new String[0];
@ -328,8 +343,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
public String getConfigDir() {
return instanceDir.resolve("conf").toString();
}
public String getDataDir() {
public String getDataDir() {
return dataDir;
}
@ -341,23 +356,28 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
* EXPERT
* <p>
* The underlying class loader. Most applications will not need to use this.
*
* @return The {@link ClassLoader}
*/
public ClassLoader getClassLoader() {
return classLoader;
}
/** Opens a schema resource by its name.
/**
* Opens a schema resource by its name.
* Override this method to customize loading schema resources.
*@return the stream for the named schema
*
* @return the stream for the named schema
*/
public InputStream openSchema(String name) throws IOException {
return openResource(name);
}
/** Opens a config resource by its name.
/**
* Opens a config resource by its name.
* Override this method to customize loading config resources.
*@return the stream for the named configuration
*
* @return the stream for the named configuration
*/
public InputStream openConfig(String name) throws IOException {
return openResource(name);
@ -372,14 +392,16 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
throw new IOException("File " + pathToCheck + " is outside resource loader dir " + instanceDir +
"; set -Dsolr.allow.unsafe.resourceloading=true to allow unsafe loading");
}
/** Opens any resource by its name.
/**
* Opens any resource by its name.
* By default, this will look in multiple locations to load the resource:
* $configDir/$resource (if resource is not absolute)
* $CWD/$resource
* otherwise, it will look for it in any jar accessible through the class loader.
* Override this method to customize loading resources.
*@return the stream for the named resource
*
* @return the stream for the named resource
*/
@Override
public InputStream openResource(String resource) throws IOException {
@ -461,22 +483,22 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
* @throws IOException If there is a low-level I/O error.
*/
public List<String> getLines(String resource,
String encoding) throws IOException {
String encoding) throws IOException {
return getLines(resource, Charset.forName(encoding));
}
public List<String> getLines(String resource, Charset charset) throws IOException{
public List<String> getLines(String resource, Charset charset) throws IOException {
try {
return WordlistLoader.getLines(openResource(resource), charset);
} catch (CharacterCodingException ex) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error loading resource (wrong encoding?): " + resource, ex);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error loading resource (wrong encoding?): " + resource, ex);
}
}
/*
* A static map of short class name to fully qualified class name
* A static map of short class name to fully qualified class name
*/
private static final Map<String, String> classNameCache = new ConcurrentHashMap<>();
@ -486,14 +508,14 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
// Using this pattern, legacy analysis components from previous Solr versions are identified and delegated to SPI loader:
private static final Pattern legacyAnalysisPattern =
Pattern.compile("((\\Q"+base+".analysis.\\E)|(\\Q"+project+".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
private static final Pattern legacyAnalysisPattern =
Pattern.compile("((\\Q" + base + ".analysis.\\E)|(\\Q" + project + ".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
@Override
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
return findClass(cname, expectedType, empty);
}
/**
* This method loads a class either with its FQN or a short-name (solr.class-simplename or class-simplename).
* It tries to load the class with the name that is given first and if it fails, it tries all the known
@ -501,25 +523,25 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
* for the same class faster. The caching is done only if the class is loaded by the webapp classloader and it
* is loaded using a shortname.
*
* @param cname The name or the short name of the class.
* @param cname The name or the short name of the class.
* @param subpackages the packages to be tried if the cname starts with solr.
* @return the loaded class. An exception is thrown if it fails
*/
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType, String... subpackages) {
if (subpackages == null || subpackages.length == 0 || subpackages == packages) {
subpackages = packages;
String c = classNameCache.get(cname);
if(c != null) {
String c = classNameCache.get(cname);
if (c != null) {
try {
return Class.forName(c, true, classLoader).asSubclass(expectedType);
} catch (ClassNotFoundException | ClassCastException e) {
// this can happen if the legacyAnalysisPattern below caches the wrong thing
log.warn("Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e);
log.warn( name + " Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e);
classNameCache.remove(cname);
}
}
}
Class<? extends T> clazz = null;
try {
// first try legacy analysis patterns, now replaced by Lucene's Analysis package:
@ -537,43 +559,43 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
} else {
log.warn("'{}' looks like an analysis factory, but caller requested different class type: {}", cname, expectedType.getName());
}
} catch (IllegalArgumentException ex) {
} catch (IllegalArgumentException ex) {
// ok, we fall back to legacy loading
}
}
// first try cname == full name
try {
return clazz = Class.forName(cname, true, classLoader).asSubclass(expectedType);
} catch (ClassNotFoundException e) {
String newName=cname;
String newName = cname;
if (newName.startsWith(project)) {
newName = cname.substring(project.length()+1);
newName = cname.substring(project.length() + 1);
}
for (String subpackage : subpackages) {
try {
String name = base + '.' + subpackage + newName;
log.trace("Trying class name " + name);
return clazz = Class.forName(name,true,classLoader).asSubclass(expectedType);
return clazz = Class.forName(name, true, classLoader).asSubclass(expectedType);
} catch (ClassNotFoundException e1) {
// ignore... assume first exception is best.
}
}
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error loading class '" + cname + "'", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, name +" Error loading class '" + cname + "'", e);
}
} finally {
if (clazz != null) {
//cache the shortname vs FQN if it is loaded by the webapp classloader and it is loaded
// using a shortname
if (clazz.getClassLoader() == SolrResourceLoader.class.getClassLoader() &&
!cname.equals(clazz.getName()) &&
(subpackages.length == 0 || subpackages == packages)) {
!cname.equals(clazz.getName()) &&
(subpackages.length == 0 || subpackages == packages)) {
//store in the cache
classNameCache.put(cname, clazz.getName());
}
// print warning if class is deprecated
if (clazz.isAnnotationPresent(Deprecated.class)) {
log.warn("Solr loaded a deprecated plugin/analysis class [{}]. Please consult documentation how to replace it accordingly.",
@ -582,9 +604,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
}
}
static final String empty[] = new String[0];
@Override
public <T> T newInstance(String name, Class<T> expectedType) {
return newInstance(name, expectedType, empty);
@ -593,33 +615,32 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
private static final Class[] NO_CLASSES = new Class[0];
private static final Object[] NO_OBJECTS = new Object[0];
public <T> T newInstance(String cname, Class<T> expectedType, String ... subpackages) {
public <T> T newInstance(String cname, Class<T> expectedType, String... subpackages) {
return newInstance(cname, expectedType, subpackages, NO_CLASSES, NO_OBJECTS);
}
public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String ... subpackages) {
public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String... subpackages) {
Class<? extends CoreAdminHandler> clazz = findClass(cname, CoreAdminHandler.class, subpackages);
if( clazz == null ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"Can not find class: "+cname + " in " + classLoader);
if (clazz == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Can not find class: " + cname + " in " + classLoader);
}
CoreAdminHandler obj = null;
try {
Constructor<? extends CoreAdminHandler> ctor = clazz.getConstructor(CoreContainer.class);
obj = ctor.newInstance(coreContainer);
}
catch (Exception e) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"Error instantiating class: '" + clazz.getName()+"'", e);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error instantiating class: '" + clazz.getName() + "'", e);
}
if (!live) {
//TODO: Does SolrCoreAware make sense here since in a multi-core context
// which core are we talking about ?
if( obj instanceof ResourceLoaderAware ) {
assertAwareCompatibility( ResourceLoaderAware.class, obj );
waitingForResources.add( (ResourceLoaderAware)obj );
if (obj instanceof ResourceLoaderAware) {
assertAwareCompatibility(ResourceLoaderAware.class, obj);
waitingForResources.add((ResourceLoaderAware) obj);
}
}
@ -627,12 +648,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
public <T> T newInstance(String cName, Class<T> expectedType, String [] subPackages, Class[] params, Object[] args){
public <T> T newInstance(String cName, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args) {
Class<? extends T> clazz = findClass(cName, expectedType, subPackages);
if( clazz == null ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"Can not find class: "+cName + " in " + classLoader);
if (clazz == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Can not find class: " + cName + " in " + classLoader);
}
T obj = null;
@ -653,25 +673,25 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
} catch (Error err) {
log.error("Loading Class " + cName + " ("+clazz.getName() + ") triggered serious java error: "
+ err.getClass().getName(), err);
log.error("Loading Class " + cName + " (" + clazz.getName() + ") triggered serious java error: "
+ err.getClass().getName(), err);
throw err;
} catch (Exception e) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"Error instantiating class: '" + clazz.getName()+"'", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error instantiating class: '" + clazz.getName() + "'", e);
}
if (!live) {
if( obj instanceof SolrCoreAware ) {
assertAwareCompatibility( SolrCoreAware.class, obj );
waitingForCore.add( (SolrCoreAware)obj );
if (obj instanceof SolrCoreAware) {
assertAwareCompatibility(SolrCoreAware.class, obj);
waitingForCore.add((SolrCoreAware) obj);
}
if( obj instanceof ResourceLoaderAware ) {
assertAwareCompatibility( ResourceLoaderAware.class, obj );
waitingForResources.add( (ResourceLoaderAware)obj );
if (obj instanceof ResourceLoaderAware) {
assertAwareCompatibility(ResourceLoaderAware.class, obj);
waitingForResources.add((ResourceLoaderAware) obj);
}
if (obj instanceof SolrInfoBean){
if (obj instanceof SolrInfoBean) {
//TODO: Assert here?
infoMBeans.add((SolrInfoBean) obj);
}
@ -680,12 +700,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
return obj;
}
/**
* Tell all {@link SolrCoreAware} instances about the SolrCore
*/
public void inform(SolrCore core)
{
public void inform(SolrCore core) {
this.dataDir = core.getDataDir();
// make a copy to avoid potential deadlock of a callback calling newInstance and trying to
@ -698,22 +717,21 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
waitingForCore.clear();
}
for( SolrCoreAware aware : arr) {
aware.inform( core );
for (SolrCoreAware aware : arr) {
aware.inform(core);
}
}
// this is the last method to be called in SolrCore before the latch is released.
live = true;
}
/**
* Tell all {@link ResourceLoaderAware} instances about the loader
*/
public void inform( ResourceLoader loader ) throws IOException
{
public void inform(ResourceLoader loader) throws IOException {
// make a copy to avoid potential deadlock of a callback adding to the list
// make a copy to avoid potential deadlock of a callback adding to the list
ResourceLoaderAware[] arr;
while (waitingForResources.size() > 0) {
@ -722,7 +740,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
waitingForResources.clear();
}
for( ResourceLoaderAware aware : arr) {
for (ResourceLoaderAware aware : arr) {
aware.inform(loader);
}
}
@ -730,6 +748,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Register any {@link SolrInfoBean}s
*
* @param infoRegistry The Info Registry
*/
public void inform(Map<String, SolrInfoBean> infoRegistry) {
@ -755,7 +774,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
}
}
}
/**
* Determines the solrhome from the environment.
* Tries JNDI (java:comp/env/solr/home) then system property (solr.solr.home);
@ -765,12 +784,13 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Finds the solrhome based on looking up the value in one of three places:
* <ol>
* <li>JNDI: via java:comp/env/solr/home</li>
* <li>The system property solr.solr.home</li>
* <li>Look in the current working directory for a solr/ directory</li>
* <li>JNDI: via java:comp/env/solr/home</li>
* <li>The system property solr.solr.home</li>
* <li>Look in the current working directory for a solr/ directory</li>
* </ol>
*
* <p>
* The return value is normalized. Normalization essentially means it ends in a trailing slash.
*
* @return A normalized solrhome
* @see #normalizeDir(String)
*/
@ -780,27 +800,27 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
// Try JNDI
try {
Context c = new InitialContext();
home = (String)c.lookup("java:comp/env/"+project+"/home");
logOnceInfo("home_using_jndi", "Using JNDI solr.home: "+home );
home = (String) c.lookup("java:comp/env/" + project + "/home");
logOnceInfo("home_using_jndi", "Using JNDI solr.home: " + home);
} catch (NoInitialContextException e) {
log.debug("JNDI not configured for "+project+" (NoInitialContextEx)");
log.debug("JNDI not configured for " + project + " (NoInitialContextEx)");
} catch (NamingException e) {
log.debug("No /"+project+"/home in JNDI");
} catch( RuntimeException ex ) {
log.debug("No /" + project + "/home in JNDI");
} catch (RuntimeException ex) {
log.warn("Odd RuntimeException while testing for JNDI: " + ex.getMessage());
}
}
// Now try system property
if( home == null ) {
if (home == null) {
String prop = project + ".solr.home";
home = System.getProperty(prop);
if( home != null ) {
logOnceInfo("home_using_sysprop", "Using system property "+prop+": " + home );
if (home != null) {
logOnceInfo("home_using_sysprop", "Using system property " + prop + ": " + home);
}
}
// if all else fails, try
if( home == null ) {
if (home == null) {
home = project + '/';
logOnceInfo("home_default", project + " home defaulted to '" + home + "' (could not find system property or JNDI)");
}
@ -809,22 +829,23 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
/**
* Solr allows users to store arbitrary files in a special directory located directly under SOLR_HOME.
*
* <p>
* This directory is generally created by each node on startup. Files located in this directory can then be
* manipulated using select Solr features (e.g. streaming expressions).
*/
public static final String USER_FILES_DIRECTORY = "userfiles";
public static void ensureUserFilesDataDir(Path solrHome) {
final Path userFilesPath = getUserFilesPath(solrHome);
final File userFilesDirectory = new File(userFilesPath.toString());
if (! userFilesDirectory.exists()) {
if (!userFilesDirectory.exists()) {
try {
final boolean created = userFilesDirectory.mkdir();
if (! created) {
if (!created) {
log.warn("Unable to create [{}] directory in SOLR_HOME [{}]. Features requiring this directory may fail.", USER_FILES_DIRECTORY, solrHome);
}
} catch (Exception e) {
log.warn("Unable to create [" + USER_FILES_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e);
log.warn("Unable to create [" + USER_FILES_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e);
}
}
}
@ -847,72 +868,73 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
public Path getInstancePath() {
return instanceDir;
}
/**
* Keep a list of classes that are allowed to implement each 'Aware' interface
*/
private static final Map<Class, Class[]> awareCompatibility;
static {
awareCompatibility = new HashMap<>();
awareCompatibility.put(
SolrCoreAware.class, new Class[] {
// DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
// VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
CodecFactory.class,
DirectoryFactory.class,
ManagedIndexSchemaFactory.class,
QueryResponseWriter.class,
SearchComponent.class,
ShardHandlerFactory.class,
SimilarityFactory.class,
SolrRequestHandler.class,
UpdateRequestProcessorFactory.class
}
awareCompatibility.put(
SolrCoreAware.class, new Class[]{
// DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
// VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
CodecFactory.class,
DirectoryFactory.class,
ManagedIndexSchemaFactory.class,
QueryResponseWriter.class,
SearchComponent.class,
ShardHandlerFactory.class,
SimilarityFactory.class,
SolrRequestHandler.class,
UpdateRequestProcessorFactory.class
}
);
awareCompatibility.put(
ResourceLoaderAware.class, new Class[] {
// DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
// VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
CharFilterFactory.class,
TokenFilterFactory.class,
TokenizerFactory.class,
QParserPlugin.class,
FieldType.class
}
ResourceLoaderAware.class, new Class[]{
// DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
// VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
CharFilterFactory.class,
TokenFilterFactory.class,
TokenizerFactory.class,
QParserPlugin.class,
FieldType.class
}
);
}
/**
* Utility function to throw an exception if the class is invalid
*/
static void assertAwareCompatibility( Class aware, Object obj )
{
Class[] valid = awareCompatibility.get( aware );
if( valid == null ) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"Unknown Aware interface: "+aware );
static void assertAwareCompatibility(Class aware, Object obj) {
Class[] valid = awareCompatibility.get(aware);
if (valid == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unknown Aware interface: " + aware);
}
for( Class v : valid ) {
if( v.isInstance( obj ) ) {
for (Class v : valid) {
if (v.isInstance(obj)) {
return;
}
}
StringBuilder builder = new StringBuilder();
builder.append( "Invalid 'Aware' object: " ).append( obj );
builder.append( " -- ").append( aware.getName() );
builder.append( " must be an instance of: " );
for( Class v : valid ) {
builder.append( "[" ).append( v.getName() ).append( "] ") ;
builder.append("Invalid 'Aware' object: ").append(obj);
builder.append(" -- ").append(aware.getName());
builder.append(" must be an instance of: ");
for (Class v : valid) {
builder.append("[").append(v.getName()).append("] ");
}
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, builder.toString() );
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, builder.toString());
}
@Override
public void close() throws IOException {
IOUtils.close(classLoader);
}
public List<SolrInfoBean> getInfoMBeans(){
public List<SolrInfoBean> getInfoMBeans() {
return Collections.unmodifiableList(infoMBeans);
}
@ -922,8 +944,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
File confFile = new File(loader.getConfigDir(), resourceName);
try {
File parentDir = confFile.getParentFile();
if ( ! parentDir.isDirectory()) {
if ( ! parentDir.mkdirs()) {
if (!parentDir.isDirectory()) {
if (!parentDir.mkdirs()) {
final String msg = "Can't create managed schema directory " + parentDir.getAbsolutePath();
log.error(msg);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);

13
solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
View File

@ -354,26 +354,21 @@ public class DistribPackageStore implements PackageStore {
Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(), url, null);
} catch (Exception e) {
log.info("Node: " + node +
" failed to respond for blob notification", e);
" failed to respond for file fetch notification", e);
//ignore the exception
// some nodes may be down or not responding
}
i++;
}
} finally {
new Thread(() -> {
coreContainer.getUpdateShardHandler().getUpdateExecutor().submit(() -> {
try {
// keep the jar in memory for 10 secs , so that
//every node can download it from memory without the file system
Thread.sleep(10 * 1000);
} catch (Exception e) {
//don't care
} finally {
tmpFiles.remove(entry.getPath());
}
}).start();
return null;
});
}
}

2
solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
View File

@ -198,7 +198,7 @@ public class PackageStoreAPI {
cryptoKeys = new CryptoKeys(keys);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error parsing public keyts in ZooKeeper");
"Error parsing public keys in ZooKeeper");
}
for (String sig : sigs) {
if (cryptoKeys.verify(sig, buf) == null) {

39
solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
View File

@ -68,6 +68,7 @@ import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
@ -150,7 +151,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
public static boolean getImmutable(SolrCore core) {
NamedList configSetProperties = core.getConfigSetProperties();
if(configSetProperties == null) return false;
if (configSetProperties == null) return false;
Object immutable = configSetProperties.get(IMMUTABLE_CONFIGSET_ARG);
return immutable != null && Boolean.parseBoolean(immutable.toString());
}
@ -245,10 +246,26 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
if (componentName != null) {
Map map = (Map) val.get(parts.get(1));
if (map != null) {
val.put(parts.get(1), makeMap(componentName, map.get(componentName)));
Object o = map.get(componentName);
val.put(parts.get(1), makeMap(componentName, o));
if (req.getParams().getBool("meta", false)) {
// meta=true is asking for the package info of the plugin
// We go through all the listeners and see if there is one registered for this plugin
List<PackageListeners.Listener> listeners = req.getCore().getPackageListeners().getListeners();
for (PackageListeners.Listener listener :
listeners) {
PluginInfo info = listener.pluginInfo();
if(info == null) continue;
if (info.type.equals(parts.get(1)) && info.name.equals(componentName)) {
if (o instanceof Map) {
Map m1 = (Map) o;
m1.put("_packageinfo_", listener.getPackageVersion());
}
}
}
}
}
}
resp.add("config", val);
}
}
@ -429,7 +446,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing params", errs);
throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, "error processing params", errs);
}
SolrResourceLoader loader = req.getCore().getResourceLoader();
@ -492,14 +509,15 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
}
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing commands", errs);
log.error("ERROR:" + Utils.toJSONString(errs));
throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, "error processing commands", errs);
}
SolrResourceLoader loader = req.getCore().getResourceLoader();
if (loader instanceof ZkSolrResourceLoader) {
int latestVersion = ZkController.persistConfigResourceToZooKeeper((ZkSolrResourceLoader) loader, overlay.getZnodeVersion(),
ConfigOverlay.RESOURCE_NAME, overlay.toByteArray(), true);
log.info("Executed config commands successfully and persisted to ZK {}", ops);
log.debug("Executed config commands successfully and persisted to ZK {}", ops);
waitForAllReplicasState(req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(),
req.getCore().getCoreContainer().getZkController(),
ConfigOverlay.NAME,
@ -530,8 +548,8 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
op.getMap(PluginInfo.INVARIANTS, null);
op.getMap(PluginInfo.APPENDS, null);
if (op.hasError()) return overlay;
if(info.clazz == PluginBag.RuntimeLib.class) {
if(!PluginBag.RuntimeLib.isEnabled()){
if (info.clazz == PluginBag.RuntimeLib.class) {
if (!PluginBag.RuntimeLib.isEnabled()) {
op.addError("Solr not started with -Denable.runtime.lib=true");
return overlay;
}
@ -563,7 +581,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
private boolean pluginExists(SolrConfig.SolrPluginInfo info, ConfigOverlay overlay, String name) {
List<PluginInfo> l = req.getCore().getSolrConfig().getPluginInfos(info.clazz.getName());
for (PluginInfo pluginInfo : l) if(name.equals( pluginInfo.name)) return true;
for (PluginInfo pluginInfo : l) if (name.equals(pluginInfo.name)) return true;
return overlay.getNamedPlugins(info.getCleanTag()).containsKey(name);
}
@ -574,6 +592,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
try {
req.getCore().createInitInstance(new PluginInfo(SolrRequestHandler.TYPE, op.getDataMap()), expected, clz, "");
} catch (Exception e) {
log.error("Error checking plugin : ", e);
op.addError(e.getMessage());
return false;
}
@ -679,7 +698,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
c == '_' ||
c == '-' ||
c == '.'
) continue;
) continue;
else {
return formatString("''{0}'' name should only have chars [a-zA-Z_-.0-9] ", s);
}

60
solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
View File

@ -40,6 +40,8 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrQueryTimeoutImpl;
@ -58,11 +60,9 @@ import static org.apache.solr.common.params.CommonParams.PATH;
/**
*
* Refer SOLR-281
*
*/
public class SearchHandler extends RequestHandlerBase implements SolrCoreAware , PluginInfoInitialized, PermissionNameProvider {
public class SearchHandler extends RequestHandlerBase implements SolrCoreAware, PluginInfoInitialized, PermissionNameProvider {
static final String INIT_COMPONENTS = "components";
static final String INIT_FIRST_COMPONENTS = "first-components";
static final String INIT_LAST_COMPONENTS = "last-components";
@ -70,22 +70,21 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected volatile List<SearchComponent> components;
private ShardHandlerFactory shardHandlerFactory ;
private ShardHandlerFactory shardHandlerFactory;
private PluginInfo shfInfo;
private SolrCore core;
protected List<String> getDefaultComponents()
{
protected List<String> getDefaultComponents() {
ArrayList<String> names = new ArrayList<>(8);
names.add( QueryComponent.COMPONENT_NAME );
names.add( FacetComponent.COMPONENT_NAME );
names.add( FacetModule.COMPONENT_NAME );
names.add( MoreLikeThisComponent.COMPONENT_NAME );
names.add( HighlightComponent.COMPONENT_NAME );
names.add( StatsComponent.COMPONENT_NAME );
names.add( DebugComponent.COMPONENT_NAME );
names.add( ExpandComponent.COMPONENT_NAME);
names.add( TermsComponent.COMPONENT_NAME);
names.add(QueryComponent.COMPONENT_NAME);
names.add(FacetComponent.COMPONENT_NAME);
names.add(FacetModule.COMPONENT_NAME);
names.add(MoreLikeThisComponent.COMPONENT_NAME);
names.add(HighlightComponent.COMPONENT_NAME);
names.add(StatsComponent.COMPONENT_NAME);
names.add(DebugComponent.COMPONENT_NAME);
names.add(ExpandComponent.COMPONENT_NAME);
names.add(TermsComponent.COMPONENT_NAME);
return names;
}
@ -94,7 +93,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
public void init(PluginInfo info) {
init(info.initArgs);
for (PluginInfo child : info.children) {
if("shardHandlerFactory".equals(child.type)){
if ("shardHandlerFactory".equals(child.type)) {
this.shfInfo = child;
break;
}
@ -113,8 +112,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
*/
@Override
@SuppressWarnings("unchecked")
public void inform(SolrCore core)
{
public void inform(SolrCore core) {
this.core = core;
List<String> c = (List<String>) initArgs.get(INIT_COMPONENTS);
Set<String> missing = new HashSet<>(core.getSearchComponents().checkContains(c));
@ -143,6 +141,32 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
});
}
if (core.getCoreContainer().isZooKeeperAware()) {
core.getPackageListeners().addListener(new PackageListeners.Listener() {
@Override
public String packageName() {
return null;
}
@Override
public PluginInfo pluginInfo() {
return null;
}
@Override
public void changed(PackageLoader.Package pkg) {
//we could optimize this by listening to only relevant packages,
// but it is not worth optimizing as these are lightweight objects
components = null;
}
@Override
public PackageLoader.Package.Version getPackageVersion() {
return null;
}
});
}
}
private void initComponents() {

383
solr/core/src/java/org/apache/solr/pkg/PackageAPI.java Normal file
View File

@ -0,0 +1,383 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.pkg;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.solr.api.Command;
import org.apache.solr.api.EndPoint;
import org.apache.solr.api.PayloadObj;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.beans.Package;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.ReflectMapWriter;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.filestore.PackageStoreAPI;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM;
import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM;
public class PackageAPI {
public static final String PACKAGES = "packages";
public final boolean enablePackages = Boolean.parseBoolean(System.getProperty("enable.packages", "false"));
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
final CoreContainer coreContainer;
private ObjectMapper mapper = new ObjectMapper();
private final PackageLoader packageLoader;
Packages pkgs;
public final Edit editAPI = new Edit();
public final Read readAPI = new Read();
public PackageAPI(CoreContainer coreContainer, PackageLoader loader) {
this.coreContainer = coreContainer;
this.packageLoader = loader;
pkgs = new Packages();
SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
try {
registerListener(zkClient);
} catch (KeeperException | InterruptedException e) {
SolrZkClient.checkInterrupted(e);
}
}
private void registerListener(SolrZkClient zkClient)
throws KeeperException, InterruptedException {
String path = SOLR_PKGS_PATH;
zkClient.exists(path,
new Watcher() {
@Override
public void process(WatchedEvent event) {
// session events are not change events, and do not remove the watcher
if (Event.EventType.None.equals(event.getType())) {
return;
}
try {
synchronized (this) {
log.debug("Updating [{}] ... ", path);
// remake watch
final Watcher thisWatch = this;
final Stat stat = new Stat();
final byte[] data = zkClient.getData(path, thisWatch, stat, true);
pkgs = readPkgsFromZk(data, stat);
packageLoader.refreshPackageConf();
}
} catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: [{}]", e.getMessage());
} catch (KeeperException e) {
log.error("A ZK error has occurred", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
log.warn("Interrupted", e);
}
}
}, true);
}
private Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException {
if (data == null || stat == null) {
stat = new Stat();
data = coreContainer.getZkController().getZkClient()
.getData(ZkStateReader.CLUSTER_PROPS, null, stat, true);
}
Packages packages = null;
if (data == null || data.length == 0) {
packages = new Packages();
} else {
try {
packages = mapper.readValue(data, Packages.class);
packages.znodeVersion = stat.getVersion();
} catch (IOException e) {
//invalid data in packages
//TODO handle properly;
return new Packages();
}
}
return packages;
}
public static class Packages implements ReflectMapWriter {
@JsonProperty
public int znodeVersion = -1;
@JsonProperty
public Map<String, List<PkgVersion>> packages = new LinkedHashMap<>();
public Packages copy() {
Packages p = new Packages();
p.znodeVersion = this.znodeVersion;
p.packages = new LinkedHashMap<>();
packages.forEach((s, versions) ->
p.packages.put(s, new ArrayList<>(versions)));
return p;
}
}
public static class PkgVersion implements ReflectMapWriter {
@JsonProperty
public String version;
@JsonProperty
public List<String> files;
public PkgVersion() {
}
public PkgVersion(Package.AddVersion addVersion) {
this.version = addVersion.version;
this.files = addVersion.files;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof PkgVersion) {
PkgVersion that = (PkgVersion) obj;
return Objects.equals(this.version, that.version)
&& Objects.equals(this.files, that.files);
}
return false;
}
}
@EndPoint(method = SolrRequest.METHOD.POST,
path = "/cluster/package",
permission = PACKAGE_EDIT_PERM)
public class Edit {
@Command(name = "refresh")
public void refresh(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<String> payload) {
String p = payload.get();
if (p == null) {
payload.addError("Package null");
return;
}
PackageLoader.Package pkg = coreContainer.getPackageLoader().getPackage(p);
if (pkg == null) {
payload.addError("No such package: " + p);
return;
}
for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) {
Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&refreshPackage=" + p,
Utils.JAVABINCONSUMER);
}
}
@Command(name = "add")
public void add(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<Package.AddVersion> payload) {
if (!checkEnabled(payload)) return;
Package.AddVersion add = payload.get();
if (add.files.isEmpty()) {
payload.addError("No files specified");
return;
}
PackageStoreAPI packageStoreAPI = coreContainer.getPackageStoreAPI();
packageStoreAPI.validateFiles(add.files, true, s -> payload.addError(s));
if (payload.hasError()) return;
Packages[] finalState = new Packages[1];
try {
coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> {
Packages packages = null;
try {
packages = bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class);
packages = packages.copy();
} catch (IOException e) {
log.error("Error deserializing packages.json", e);
packages = new Packages();
}
packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add));
packages.znodeVersion = stat.getVersion() + 1;
finalState[0] = packages;
return Utils.toJSON(packages);
});
} catch (KeeperException | InterruptedException e) {
finalState[0] = null;
handleZkErr(e);
}
if (finalState[0] != null) {
// succeeded in updating
pkgs = finalState[0];
notifyAllNodesToSync(pkgs.znodeVersion);
packageLoader.refreshPackageConf();
}
}
@Command(name = "delete")
public void del(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<Package.DelVersion> payload) {
if (!checkEnabled(payload)) return;
Package.DelVersion delVersion = payload.get();
try {
coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> {
Packages packages = null;
try {
packages = mapper.readValue(bytes, Packages.class);
packages = packages.copy();
} catch (IOException e) {
packages = new Packages();
}
List<PkgVersion> versions = packages.packages.get(delVersion.pkg);
if (versions == null || versions.isEmpty()) {
payload.addError("No such package: " + delVersion.pkg);
return null;// no change
}
int idxToremove = -1;
for (int i = 0; i < versions.size(); i++) {
if (Objects.equals(versions.get(i).version, delVersion.version)) {
idxToremove = i;
break;
}
}
if (idxToremove == -1) {
payload.addError("No such version: " + delVersion.version);
return null;
}
versions.remove(idxToremove);
packages.znodeVersion = stat.getVersion() + 1;
return Utils.toJSON(packages);
});
} catch (KeeperException | InterruptedException e) {
handleZkErr(e);
}
}
}
private boolean checkEnabled(CommandOperation payload) {
if (!enablePackages) {
payload.addError("Package loading is not enabled , Start your nodes with -Denable.packages=true");
return false;
}
return true;
}
@EndPoint(
method = SolrRequest.METHOD.GET,
path = {"/cluster/package/",
"/cluster/package/{name}"},
permission = PACKAGE_READ_PERM
)
public class Read {
@Command()
public void get(SolrQueryRequest req, SolrQueryResponse rsp) {
String refresh = req.getParams().get("refreshPackage");
if (refresh != null) {
packageLoader.notifyListeners(refresh);
return;
}
int expectedVersion = req.getParams().getInt("expectedVersion", -1);
if (expectedVersion != -1) {
syncToVersion(expectedVersion);
}
String name = req.getPathTemplateValues().get("name");
if (name == null) {
rsp.add("result", pkgs);
} else {
rsp.add("result", Collections.singletonMap(name, pkgs.packages.get(name)));
}
}
private void syncToVersion(int expectedVersion) {
int origVersion = pkgs.znodeVersion;
for (int i = 0; i < 10; i++) {
log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion);
if (pkgs.znodeVersion >= expectedVersion) {
if (origVersion < pkgs.znodeVersion) {
packageLoader.refreshPackageConf();
}
return;
}
try {
Thread.sleep(10);
} catch (InterruptedException e) {
}
try {
pkgs = readPkgsFromZk(null, null);
} catch (KeeperException | InterruptedException e) {
handleZkErr(e);
}
}
}
}
void notifyAllNodesToSync(int expected) {
for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) {
Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&expectedVersion" + expected,
Utils.JAVABINCONSUMER);
}
}
public void handleZkErr(Exception e) {
log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e));
}
}

111
solr/core/src/java/org/apache/solr/pkg/PackageListeners.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.pkg;
import java.lang.invoke.MethodHandles;
import java.lang.ref.Reference;
import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.logging.MDCLoggingContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PackageListeners {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String PACKAGE_VERSIONS = "PKG_VERSIONS";
private SolrCore core;
public PackageListeners(SolrCore core) {
this.core = core;
}
// this registry only keeps a weak reference because it does not want to
// cause a memory leak if the listener forgets to unregister itself
private List<Reference<Listener>> listeners = new ArrayList<>();
public synchronized void addListener(Listener listener) {
listeners.add(new SoftReference<>(listener));
}
public synchronized void removeListener(Listener listener) {
Iterator<Reference<Listener>> it = listeners.iterator();
while (it.hasNext()) {
Reference<Listener> ref = it.next();
Listener pkgListener = ref.get();
if (pkgListener == null || pkgListener == listener) {
it.remove();
}
}
}
synchronized void packagesUpdated(List<PackageLoader.Package> pkgs) {
if(core != null) MDCLoggingContext.setCore(core);
try {
for (PackageLoader.Package pkgInfo : pkgs) {
invokeListeners(pkgInfo);
}
} finally {
if(core != null) MDCLoggingContext.clear();
}
}
private synchronized void invokeListeners(PackageLoader.Package pkg) {
for (Reference<Listener> ref : listeners) {
Listener listener = ref.get();
if(listener == null) continue;
if (listener.packageName() == null || listener.packageName().equals(pkg.name())) {
listener.changed(pkg);
}
}
}
public List<Listener> getListeners() {
List<Listener> result = new ArrayList<>();
for (Reference<Listener> ref : listeners) {
Listener l = ref.get();
if (l != null) {
result.add(l);
}
}
return result;
}
public interface Listener {
/**Name of the package or null to loisten to all package changes
*/
String packageName();
PluginInfo pluginInfo();
void changed(PackageLoader.Package pkg);
PackageLoader.Package.Version getPackageVersion();
}
}

276
solr/core/src/java/org/apache/solr/pkg/PackageLoader.java Normal file
View File

@ -0,0 +1,276 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.pkg;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.MalformedURLException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The class that holds a mapping of various packages and classloaders
*/
public class PackageLoader {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final CoreContainer coreContainer;
private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
private PackageAPI.Packages myCopy;
private PackageAPI packageAPI;
public PackageLoader(CoreContainer coreContainer) {
this.coreContainer = coreContainer;
packageAPI = new PackageAPI(coreContainer, this);
myCopy = packageAPI.pkgs;
}
public PackageAPI getPackageAPI() {
return packageAPI;
}
public Package getPackage(String key) {
return packageClassLoaders.get(key);
}
public Map<String, Package> getPackages() {
return Collections.EMPTY_MAP;
}
public void refreshPackageConf() {
log.info("{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion);
List<Package> updated = new ArrayList<>();
Map<String, List<PackageAPI.PkgVersion>> modified = getModified(myCopy, packageAPI.pkgs);
for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : modified.entrySet()) {
if (e.getValue() != null) {
Package p = packageClassLoaders.get(e.getKey());
if (e.getValue() != null && p == null) {
packageClassLoaders.put(e.getKey(), p = new Package(e.getKey()));
}
p.updateVersions(e.getValue());
updated.add(p);
} else {
Package p = packageClassLoaders.remove(e.getKey());
if (p != null) {
//other classes are holding to a reference to this objecec
// they should know that this is removed
p.markDeleted();
}
}
}
for (SolrCore core : coreContainer.getCores()) {
core.getPackageListeners().packagesUpdated(updated);
}
}
public Map<String, List<PackageAPI.PkgVersion>> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) {
Map<String, List<PackageAPI.PkgVersion>> changed = new HashMap<>();
for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : newPkgs.packages.entrySet()) {
List<PackageAPI.PkgVersion> versions = old.packages.get(e.getKey());
if (versions != null) {
if (!Objects.equals(e.getValue(), versions)) {
log.info("Package {} is modified ", e.getKey());
changed.put(e.getKey(), e.getValue());
}
} else {
log.info("A new package: {} introduced", e.getKey());
changed.put(e.getKey(), e.getValue());
}
}
//some packages are deleted altogether
for (String s : old.packages.keySet()) {
if (!newPkgs.packages.keySet().contains(s)) {
log.info("Package: {} is removed althogether", s);
changed.put(s, null);
}
}
return changed;
}
public void notifyListeners(String pkg) {
Package p = packageClassLoaders.get(pkg);
if(p != null){
List<Package> l = Collections.singletonList(p);
for (SolrCore core : coreContainer.getCores()) {
core.getPackageListeners().packagesUpdated(l);
}
}
}
/**
* represents a package definition in the packages.json
*/
public class Package {
final String name;
final Map<String, Version> myVersions = new ConcurrentHashMap<>();
private List<String> sortedVersions = new CopyOnWriteArrayList<>();
String latest;
private boolean deleted;
Package(String name) {
this.name = name;
}
public boolean isDeleted() {
return deleted;
}
private synchronized void updateVersions(List<PackageAPI.PkgVersion> modified) {
for (PackageAPI.PkgVersion v : modified) {
Version version = myVersions.get(v.version);
if (version == null) {
log.info("A new version: {} added for package: {} with artifacts {}", v.version, this.name, v.files);
myVersions.put(v.version, new Version(this, v));
sortedVersions.add(v.version);
}
}
Set<String> newVersions = new HashSet<>();
for (PackageAPI.PkgVersion v : modified) {
newVersions.add(v.version);
}
for (String s : new HashSet<>(myVersions.keySet())) {
if (!newVersions.contains(s)) {
log.info("version: {} is removed from package: {}", s, this.name);
sortedVersions.remove(s);
myVersions.remove(s);
}
}
sortedVersions.sort(String::compareTo);
if (sortedVersions.size() > 0) {
String latest = sortedVersions.get(sortedVersions.size() - 1);
if (!latest.equals(this.latest)) {
log.info("version: {} is the new latest in package: {}", latest, this.name);
}
this.latest = latest;
} else {
log.error("latest version: null");
latest = null;
}
}
public Version getLatest() {
return latest == null ? null : myVersions.get(latest);
}
public Version getLatest(String lessThan) {
if (lessThan == null) {
return getLatest();
}
String latest = findBiggest(lessThan, new ArrayList(sortedVersions));
return latest == null ? null : myVersions.get(latest);
}
public String name() {
return name;
}
private void markDeleted() {
deleted = true;
myVersions.clear();
sortedVersions.clear();
latest = null;
}
public class Version implements MapWriter {
private final Package parent;
private SolrResourceLoader loader;
private final PackageAPI.PkgVersion version;
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put("package", parent.name());
version.writeMap(ew);
}
Version(Package parent, PackageAPI.PkgVersion v) {
this.parent = parent;
this.version = v;
List<Path> paths = new ArrayList<>();
for (String file : version.files) {
paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
}
try {
loader = new SolrResourceLoader(
"PACKAGE_LOADER: " + parent.name() + ":" + version,
paths,
coreContainer.getResourceLoader().getInstancePath(),
coreContainer.getResourceLoader().getClassLoader());
} catch (MalformedURLException e) {
log.error("Could not load classloader ", e);
}
}
public String getVersion() {
return version.version;
}
public Collection getFiles() {
return Collections.unmodifiableList(version.files);
}
public SolrResourceLoader getLoader() {
return loader;
}
}
}
private static String findBiggest(String lessThan, List<String> sortedList) {
String latest = null;
for (String v : sortedList) {
if (v.compareTo(lessThan) < 1) {
latest = v;
} else break;
}
return latest;
}
}

123
solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java Normal file
View File

@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.pkg;
import java.lang.invoke.MethodHandles;
import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String LATEST = "$LATEST";
private final SolrCore core;
private final SolrConfig.SolrPluginInfo pluginMeta;
private PackageLoader.Package.Version pkgVersion;
private PluginInfo info;
public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
super(info);
this.core = core;
this.pluginMeta = pluginMeta;
this.info = info;
reload(core.getCoreContainer().getPackageLoader().getPackage(info.pkgName));
core.getPackageListeners().addListener(new PackageListeners.Listener() {
@Override
public String packageName() {
return info.pkgName;
}
@Override
public PluginInfo pluginInfo() {
return info;
}
@Override
public void changed(PackageLoader.Package pkg) {
reload(pkg);
}
@Override
public PackageLoader.Package.Version getPackageVersion() {
return pkgVersion;
}
});
}
private String maxVersion() {
RequestParams.ParamSet p = core.getSolrConfig().getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS);
if (p == null) {
return null;
}
Object o = p.get().get(info.pkgName);
if (o == null || LATEST.equals(o)) return null;
return o.toString();
}
private synchronized void reload(PackageLoader.Package pkg) {
String lessThan = maxVersion();
PackageLoader.Package.Version newest = pkg.getLatest(lessThan);
if (newest == null) {
log.error("No latest version available for package : {}", pkg.name());
return;
}
if (lessThan != null) {
PackageLoader.Package.Version pkgLatest = pkg.getLatest();
if (pkgLatest != newest) {
log.info("Using version :{}. latest is {}, params.json has config {} : {}", newest.getVersion(), pkgLatest.getVersion(), pkg.name(), lessThan);
}
}
if (pkgVersion != null) {
if (newest == pkgVersion) {
//I'm already using the latest classloder in the package. nothing to do
return;
}
}
log.info("loading plugin: {} -> {} using package {}:{}",
pluginInfo.type, pluginInfo.name, pkg.name(), newest.getVersion());
Object instance = SolrCore.createInstance(pluginInfo.className,
pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
PluginBag.initInstance(instance, pluginInfo);
T old = inst;
inst = (T) instance;
pkgVersion = newest;
if (old instanceof AutoCloseable) {
AutoCloseable closeable = (AutoCloseable) old;
try {
closeable.close();
} catch (Exception e) {
log.error("error closing plugin", e);
}
}
}
}

3
solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
View File

@ -53,6 +53,9 @@ public interface PermissionNameProvider {
METRICS_HISTORY_READ_PERM("metrics-history-read", null),
FILESTORE_READ_PERM("filestore-read", null),
FILESTORE_WRITE_PERM("filestore-write", null),
PACKAGE_EDIT_PERM("package-edit", null),
PACKAGE_READ_PERM("package-read", null),
ALL("all", unmodifiableSet(new HashSet<>(asList("*", null))))
;
final String name;

43
solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
View File

@ -23,6 +23,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import com.google.common.collect.ImmutableMap;
import org.apache.solr.common.SolrException;
@ -33,9 +34,12 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.pkg.PackagePluginHolder;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
@ -126,8 +130,7 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
// wrap in an ArrayList so we know we know we can do fast index lookups
// and that add(int,Object) is supported
List<UpdateRequestProcessorFactory> list = new ArrayList<>
(solrCore.initPlugins(info.getChildren("processor"),UpdateRequestProcessorFactory.class,null));
List<UpdateRequestProcessorFactory> list = createProcessors(info);
if(list.isEmpty()){
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@ -170,6 +173,23 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
}
private List<UpdateRequestProcessorFactory> createProcessors(PluginInfo info) {
List<PluginInfo> processors = info.getChildren("processor");
return processors.stream().map(it -> {
if(it.pkgName == null){
return solrCore.createInitInstance(it, UpdateRequestProcessorFactory.class,
UpdateRequestProcessorFactory.class.getSimpleName(), null);
} else {
return new LazyUpdateRequestProcessorFactory(new PackagePluginHolder(
it,
solrCore,
SolrConfig.classVsSolrPluginInfo.get(UpdateRequestProcessorFactory.class.getName())));
}
}).collect(Collectors.toList());
}
/**
* Creates a chain backed directly by the specified list. Modifications to
* the array will affect future calls to <code>createProcessor</code>
@ -328,7 +348,7 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
public static class LazyUpdateProcessorFactoryHolder extends PluginBag.PluginHolder<UpdateRequestProcessorFactory> {
private volatile UpdateRequestProcessorFactory lazyFactory;
public LazyUpdateProcessorFactoryHolder(final PluginBag.LazyPluginHolder holder) {
public LazyUpdateProcessorFactoryHolder(final PluginBag.PluginHolder holder) {
super(holder.getPluginInfo());
lazyFactory = new LazyUpdateRequestProcessorFactory(holder);
}
@ -339,27 +359,20 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
return lazyFactory;
}
public class LazyUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory {
private final PluginBag.LazyPluginHolder holder;
UpdateRequestProcessorFactory delegate;
public static class LazyUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory {
private final PluginBag.PluginHolder<UpdateRequestProcessorFactory> holder;
public LazyUpdateRequestProcessorFactory(PluginBag.LazyPluginHolder holder) {
public LazyUpdateRequestProcessorFactory(PluginBag.PluginHolder holder) {
this.holder = holder;
}
public UpdateRequestProcessorFactory getDelegate() {
return delegate;
return holder.get();
}
@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
if (delegate != null) return delegate.getInstance(req, rsp, next);
synchronized (this) {
if (delegate == null)
delegate = (UpdateRequestProcessorFactory) holder.get();
}
return delegate.getInstance(req, rsp, next);
return holder.get().getInstance(req, rsp, next);
}
}
}

495
solr/core/src/test/org/apache/solr/pkg/TestPackages.java Normal file
View File

@ -0,0 +1,495 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.pkg;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.Callable;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.request.beans.Package;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.ConfigRequest;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.MapWriterMap;
import org.apache.solr.common.NavigableObject;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.filestore.TestDistribPackageStore;
import org.apache.solr.util.LogLevel;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.Stat;
import org.junit.Test;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
public class TestPackages extends SolrCloudTestCase {
@Test
public void testPluginLoading() throws Exception {
System.setProperty("enable.packages", "true");
MiniSolrCloudCluster cluster =
configureCluster(4)
.withJettyConfig(jetty -> jetty.enableV2(true))
.addConfig("conf", configset("cloud-minimal"))
.configure();
try {
String FILE1 = "/mypkg/runtimelibs.jar";
String FILE2 = "/mypkg/runtimelibs_v2.jar";
String FILE3 = "/mypkg/runtimelibs_v3.jar";
String COLLECTION_NAME = "testPluginLoadingColl";
byte[] derFile = readFile("cryptokeys/pub_key512.der");
cluster.getZkClient().makePath("/keys/exe", true);
cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
"L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
Package.AddVersion add = new Package.AddVersion();
add.version = "1.0";
add.pkg = "mypkg";
add.files = Arrays.asList(new String[]{FILE1});
V2Request req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
.withPayload(Collections.singletonMap("add", add))
.build();
req.process(cluster.getSolrClient());
CollectionAdminRequest
.createCollection(COLLECTION_NAME, "conf", 2, 2)
.setMaxShardsPerNode(100)
.process(cluster.getSolrClient());
cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/cluster/package").
withMethod(SolrRequest.METHOD.GET)
.build().process(cluster.getSolrClient()),
Utils.makeMap(
":result:packages:mypkg[0]:version", "1.0",
":result:packages:mypkg[0]:files[0]", FILE1
));
String payload = "{\n" +
"'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler' }," +
"'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent' }," +
"'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter' }" +
"}";
cluster.getSolrClient().request(new ConfigRequest(payload) {
@Override
public String getCollection() {
return COLLECTION_NAME;
}
});
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "queryResponseWriter", "json1",
"mypkg", "1.0" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "searchComponent", "get",
"mypkg", "1.0" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "1.0" );
executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()),
Utils.JAVABINCONSUMER,
Utils.makeMap("class", "org.apache.solr.core.RuntimeLibReqHandler"));
executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
Utils.JSONCONSUMER,
Utils.makeMap("class", "org.apache.solr.core.RuntimeLibSearchComponent",
"Version","1"));
executeReq( "/" + COLLECTION_NAME + "/runtime?wt=json1", cluster.getRandomJetty(random()),
Utils.JSONCONSUMER,
Utils.makeMap("wt", "org.apache.solr.core.RuntimeLibResponseWriter"));
//now upload the second jar
postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2,
"j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
//add the version using package API
add.version = "1.1";
add.files = Arrays.asList(new String[]{FILE2});
req.process(cluster.getSolrClient());
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "queryResponseWriter", "json1",
"mypkg", "1.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "searchComponent", "get",
"mypkg", "1.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "1.1" );
executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
Utils.JSONCONSUMER,
Utils.makeMap( "Version","2"));
//now upload the third jar
postFileAndWait(cluster, "runtimecode/runtimelibs_v3.jar.bin", FILE3,
"a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q==");
add.version = "2.1";
add.files = Arrays.asList(new String[]{FILE3});
req.process(cluster.getSolrClient());
//now let's verify that the classes are updated
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "queryResponseWriter", "json1",
"mypkg", "2.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "searchComponent", "get",
"mypkg", "2.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "2.1" );
executeReq( "/" + COLLECTION_NAME + "/runtime?wt=json", cluster.getRandomJetty(random()),
Utils.JSONCONSUMER,
Utils.makeMap("Version","2"));
Package.DelVersion delVersion = new Package.DelVersion();
delVersion.pkg = "mypkg";
delVersion.version = "1.0";
V2Request delete = new V2Request.Builder("/cluster/package")
.withMethod(SolrRequest.METHOD.POST)
.forceV2(true)
.withPayload(Collections.singletonMap("delete", delVersion))
.build();
delete.process(cluster.getSolrClient());
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "queryResponseWriter", "json1",
"mypkg", "2.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "searchComponent", "get",
"mypkg", "2.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "2.1" );
// now remove the hughest version. So, it will roll back to the next highest one
delVersion.version = "2.1";
delete.process(cluster.getSolrClient());
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "queryResponseWriter", "json1",
"mypkg", "1.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "searchComponent", "get",
"mypkg", "1.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "1.1" );
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("collection", COLLECTION_NAME);
new GenericSolrRequest(SolrRequest.METHOD.POST, "/config/params", params ){
@Override
public RequestWriter.ContentWriter getContentWriter(String expectedType) {
return new RequestWriter.StringPayloadContentWriter("{set:{PKG_VERSIONS:{mypkg : '1.1'}}}",
ClientUtils.TEXT_JSON);
}
}.process(cluster.getSolrClient()) ;
add.version = "2.1";
add.files = Arrays.asList(new String[]{FILE3});
req.process(cluster.getSolrClient());
//the collections mypkg is set to use version 1.1
//so no upgrade
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "queryResponseWriter", "json1",
"mypkg", "1.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "searchComponent", "get",
"mypkg", "1.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "1.1" );
new GenericSolrRequest(SolrRequest.METHOD.POST, "/config/params", params ){
@Override
public RequestWriter.ContentWriter getContentWriter(String expectedType) {
return new RequestWriter.StringPayloadContentWriter("{set:{PKG_VERSIONS:{mypkg : '2.1'}}}",
ClientUtils.TEXT_JSON);
}
}.process(cluster.getSolrClient()) ;
//now, let's force every collection using 'mypkg' to refresh
//so that it uses version 2.1
new V2Request.Builder("/cluster/package")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{refresh : mypkg}")
.forceV2(true)
.build()
.process(cluster.getSolrClient());
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "queryResponseWriter", "json1",
"mypkg", "2.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "searchComponent", "get",
"mypkg", "2.1" );
verifyCmponent(cluster.getSolrClient(),
COLLECTION_NAME, "requestHandler", "/runtime",
"mypkg", "2.1" );
} finally {
cluster.shutdown();
}
}
private void executeReq(String uri, JettySolrRunner jetty, Utils.InputStreamConsumer parser, Map expected) throws Exception {
try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()){
TestDistribPackageStore.assertResponseValues(10,
() -> {
Object o = Utils.executeGET(client.getHttpClient(),
jetty.getBaseUrl() + uri, parser);
if(o instanceof NavigableObject) return (NavigableObject) o;
if(o instanceof Map) return new MapWriterMap((Map) o);
throw new RuntimeException("Unknown response");
}, expected);
}
}
private void verifyCmponent(SolrClient client, String COLLECTION_NAME,
String componentType, String componentName, String pkg, String version) throws Exception {
SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
WT, JAVABIN,
"componentName", componentName,
"meta", "true"));
String s = "queryResponseWriter";
GenericSolrRequest req1 = new GenericSolrRequest(SolrRequest.METHOD.GET,
"/config/" + componentType, params);
TestDistribPackageStore.assertResponseValues(10,
client,
req1, Utils.makeMap(
":config:" + componentType + ":" + componentName + ":_packageinfo_:package", pkg,
":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version
));
}
@Test
public void testAPI() throws Exception {
System.setProperty("enable.packages", "true");
MiniSolrCloudCluster cluster =
configureCluster(4)
.withJettyConfig(jetty -> jetty.enableV2(true))
.addConfig("conf", configset("cloud-minimal"))
.configure();
try {
String errPath = "/error/details[0]/errorMessages[0]";
String FILE1 = "/mypkg/v.0.12/jar_a.jar";
String FILE2 = "/mypkg/v.0.12/jar_b.jar";
String FILE3 = "/mypkg/v.0.13/jar_a.jar";
Package.AddVersion add = new Package.AddVersion();
add.version = "0.12";
add.pkg = "test_pkg";
add.files = Arrays.asList(new String[]{FILE1, FILE2});
V2Request req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
.withPayload(Collections.singletonMap("add", add))
.build();
//the files is not yet there. The command should fail with error saying "No such file"
expectError(req, cluster.getSolrClient(), errPath, "No such file :");
//post the jar file. No signature is sent
postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null);
add.files = Arrays.asList(new String[]{FILE1});
expectError(req, cluster.getSolrClient(), errPath,
FILE1 + " has no signature");
//now we upload the keys
byte[] derFile = readFile("cryptokeys/pub_key512.der");
cluster.getZkClient().makePath("/keys/exe", true);
cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
//and upload the same file with a different name but it has proper signature
postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE2,
"L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
// with correct signature
//after uploading the file, let's delete the keys to see if we get proper error message
cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true);
add.files = Arrays.asList(new String[]{FILE2});
expectError(req, cluster.getSolrClient(), errPath,
"ZooKeeper does not have any public keys");
//Now lets' put the keys back
cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
//this time we have a file with proper signature, public keys are in ZK
// so the add {} command should succeed
req.process(cluster.getSolrClient());
//Now verify the data in ZK
TestDistribPackageStore.assertResponseValues(1,
() -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
null, new Stat(), true))),
Utils.makeMap(
":packages:test_pkg[0]:version", "0.12",
":packages:test_pkg[0]:files[0]", FILE1
));
//post a new jar with a proper signature
postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE3,
"j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
//this time we are adding the second version of the package (0.13)
add.version = "0.13";
add.pkg = "test_pkg";
add.files = Arrays.asList(new String[]{FILE3});
//this request should succeed
req.process(cluster.getSolrClient());
//no verify the data (/packages.json) in ZK
TestDistribPackageStore.assertResponseValues(1,
() -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
null, new Stat(), true))),
Utils.makeMap(
":packages:test_pkg[1]:version", "0.13",
":packages:test_pkg[1]:files[0]", FILE3
));
//Now we will just delete one version
Package.DelVersion delVersion = new Package.DelVersion();
delVersion.version = "0.1";//this version does not exist
delVersion.pkg = "test_pkg";
req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
.withPayload(Collections.singletonMap("delete", delVersion))
.build();
//we are expecting an error
expectError(req, cluster.getSolrClient(), errPath, "No such version:");
delVersion.version = "0.12";//correct version. Should succeed
req.process(cluster.getSolrClient());
//Verify with ZK that the data is correcy
TestDistribPackageStore.assertResponseValues(1,
() -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
null, new Stat(), true))),
Utils.makeMap(
":packages:test_pkg[0]:version", "0.13",
":packages:test_pkg[0]:files[0]", FILE2
));
//So far we have been verifying the details with ZK directly
//use the package read API to verify with each node that it has the correct data
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
String path = jetty.getBaseUrl().toString().replace("/solr", "/api") + "/cluster/package?wt=javabin";
TestDistribPackageStore.assertResponseValues(10, new Callable<NavigableObject>() {
@Override
public NavigableObject call() throws Exception {
try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) {
return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), path, Utils.JAVABINCONSUMER);
}
}
}, Utils.makeMap(
":result:packages:test_pkg[0]:version", "0.13",
":result:packages:test_pkg[0]:files[0]", FILE3
));
}
} finally {
cluster.shutdown();
}
}
static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception {
ByteBuffer fileContent = getFileContent(fname);
String sha512 = DigestUtils.sha512Hex(fileContent.array());
TestDistribPackageStore.postFile(cluster.getSolrClient(),
fileContent,
path, sig);// has file, but no signature
TestDistribPackageStore.waitForAllNodesHaveFile(cluster, path, Utils.makeMap(
":files:" + path + ":sha512",
sha512
), false);
}
private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException {
try {
req.process(client);
fail("should have failed with message : " + expectErrorMsg);
} catch (BaseHttpSolrClient.RemoteExecutionException e) {
String msg = e.getMetaData()._getStr(errPath, "");
assertTrue("should have failed with message: " + expectErrorMsg + "actual message : " + msg,
msg.contains(expectErrorMsg)
);
}
}
}

2
solr/core/src/test/org/apache/solr/update/processor/RuntimeUrp.java
View File

@ -31,7 +31,7 @@ public class RuntimeUrp extends SimpleUpdateProcessorFactory {
List<String> names = new ArrayList<>();
for (UpdateRequestProcessorFactory p : processorChain.getProcessors()) {
if (p instanceof UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) {
p = ((UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) p).delegate;
p = ((UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder.LazyUpdateRequestProcessorFactory) p).getDelegate();
}
names.add(p.getClass().getSimpleName());
}

46
solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/Package.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.request.beans;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.solr.common.util.ReflectMapWriter;
/**Just a container class for POJOs used in Package APIs
*
*/
public class Package {
public static class AddVersion implements ReflectMapWriter {
@JsonProperty(value = "package", required = true)
public String pkg;
@JsonProperty(required = true)
public String version;
@JsonProperty(required = true)
public List<String> files;
}
public static class DelVersion implements ReflectMapWriter {
@JsonProperty(value = "package", required = true)
public String pkg;
@JsonProperty(required = true)
public String version;
}
}

23
solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/pakage-info.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Data objects used in V2 Requests with jackson bindings
*/
package org.apache.solr.client.solrj.request.beans;

9
solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
View File

@ -34,6 +34,7 @@ import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@ -353,6 +354,10 @@ public class SolrZkClient implements Closeable {
}
public void atomicUpdate(String path, Function<byte[], byte[]> editor) throws KeeperException, InterruptedException {
atomicUpdate(path, (stat, bytes) -> editor.apply(bytes));
}
public void atomicUpdate(String path, BiFunction<Stat , byte[], byte[]> editor) throws KeeperException, InterruptedException {
for (; ; ) {
byte[] modified = null;
byte[] zkData = null;
@ -360,7 +365,7 @@ public class SolrZkClient implements Closeable {
try {
if (exists(path, true)) {
zkData = getData(path, null, s, true);
modified = editor.apply(zkData);
modified = editor.apply(s, zkData);
if (modified == null) {
//no change , no need to persist
return;
@ -368,7 +373,7 @@ public class SolrZkClient implements Closeable {
setData(path, modified, s.getVersion(), true);
break;
} else {
modified = editor.apply(null);
modified = editor.apply(s,null);
if (modified == null) {
//no change , no need to persist
return;

1
solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
View File

@ -118,6 +118,7 @@ public class ZkStateReader implements SolrCloseable {
public static final String SOLR_AUTOSCALING_TRIGGER_STATE_PATH = "/autoscaling/triggerState";
public static final String SOLR_AUTOSCALING_NODE_ADDED_PATH = "/autoscaling/nodeAdded";
public static final String SOLR_AUTOSCALING_NODE_LOST_PATH = "/autoscaling/nodeLost";
public static final String SOLR_PKGS_PATH = "/packages.json";
public static final String DEFAULT_SHARD_PREFERENCES = "defaultShardPreferences";
public static final String REPLICATION_FACTOR = "replicationFactor";

2
solr/solrj/src/java/org/apache/solr/common/util/ReflectMapWriter.java
View File

@ -23,7 +23,7 @@ import java.lang.reflect.Modifier;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.solr.common.MapWriter;
// An implementation of MapWriter which is annotated with Jackson annotations
public interface ReflectMapWriter extends MapWriter {
@Override