SOLR-14151 Make schema components load from packages (#1669)

This commit is contained in:
Noble Paul 2020-07-16 16:05:24 +10:00 committed by GitHub
parent 9c2e7819ee
commit 03d658a7bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 663 additions and 154 deletions

View File

@ -108,7 +108,8 @@ Consult the lucene/CHANGES.txt file for additional, low level, changes in this r
New Features New Features
--------------------- ---------------------
(No changes)
* SOLR-14151 Make schema components load from packages (noble)
Improvements Improvements
--------------------- ---------------------

View File

@ -36,12 +36,12 @@ import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.JsonProperty; import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.cloud.ClusterPropertiesListener; import org.apache.solr.common.cloud.ClusterPropertiesListener;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.PathTrie; import org.apache.solr.common.util.PathTrie;
import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.ReflectMapWriter;
import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.handler.admin.ContainerPluginsApi; import org.apache.solr.handler.admin.ContainerPluginsApi;
import org.apache.solr.pkg.PackageLoader; import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
@ -225,12 +225,12 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri
@SuppressWarnings({"unchecked","rawtypes"}) @SuppressWarnings({"unchecked","rawtypes"})
public ApiInfo(PluginMeta info, List<String> errs) { public ApiInfo(PluginMeta info, List<String> errs) {
this.info = info; this.info = info;
Pair<String, String> klassInfo = org.apache.solr.core.PluginInfo.parseClassName(info.klass); PluginInfo.ClassName klassInfo = new PluginInfo.ClassName(info.klass);
pkg = klassInfo.first(); pkg = klassInfo.pkg;
if (pkg != null) { if (pkg != null) {
PackageLoader.Package p = coreContainer.getPackageLoader().getPackage(pkg); PackageLoader.Package p = coreContainer.getPackageLoader().getPackage(pkg);
if (p == null) { if (p == null) {
errs.add("Invalid package " + klassInfo.first()); errs.add("Invalid package " + klassInfo.pkg);
return; return;
} }
this.pkgVersion = p.getVersion(info.version); this.pkgVersion = p.getVersion(info.version);
@ -239,7 +239,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri
return; return;
} }
try { try {
klas = pkgVersion.getLoader().findClass(klassInfo.second(), Object.class); klas = pkgVersion.getLoader().findClass(klassInfo.className, Object.class);
} catch (Exception e) { } catch (Exception e) {
log.error("Error loading class", e); log.error("Error loading class", e);
errs.add("Error loading class " + e.toString()); errs.add("Error loading class " + e.toString());
@ -247,7 +247,7 @@ public class CustomContainerPlugins implements ClusterPropertiesListener, MapWri
} }
} else { } else {
try { try {
klas = Class.forName(klassInfo.second()); klas = Class.forName(klassInfo.className);
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
errs.add("Error loading class " + e.toString()); errs.add("Error loading class " + e.toString());
return; return;

View File

@ -19,6 +19,7 @@ package org.apache.solr.core;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.IndexSchema;
/** /**
* Stores a core's configuration in the form of a SolrConfig and IndexSchema. * Stores a core's configuration in the form of a SolrConfig and IndexSchema.
* Immutable. * Immutable.
@ -30,7 +31,7 @@ public class ConfigSet {
private final SolrConfig solrconfig; private final SolrConfig solrconfig;
private final IndexSchema indexSchema; private final SchemaSupplier schemaSupplier;
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
private final NamedList properties; private final NamedList properties;
@ -38,11 +39,11 @@ public class ConfigSet {
private final boolean trusted; private final boolean trusted;
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
public ConfigSet(String name, SolrConfig solrConfig, IndexSchema indexSchema, public ConfigSet(String name, SolrConfig solrConfig, SchemaSupplier indexSchemaSupplier,
NamedList properties, boolean trusted) { NamedList properties, boolean trusted) {
this.name = name; this.name = name;
this.solrconfig = solrConfig; this.solrconfig = solrConfig;
this.indexSchema = indexSchema; this.schemaSupplier = indexSchemaSupplier;
this.properties = properties; this.properties = properties;
this.trusted = trusted; this.trusted = trusted;
} }
@ -55,8 +56,15 @@ public class ConfigSet {
return solrconfig; return solrconfig;
} }
/**
*
* @param forceFetch get a fresh value and not cached value
*/
public IndexSchema getIndexSchema(boolean forceFetch) {
return schemaSupplier.get(forceFetch);
}
public IndexSchema getIndexSchema() { public IndexSchema getIndexSchema() {
return indexSchema; return schemaSupplier.get(false);
} }
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
@ -67,4 +75,14 @@ public class ConfigSet {
public boolean isTrusted() { public boolean isTrusted() {
return trusted; return trusted;
} }
/**Provide a Schema object on demand
* We want IndexSchema Objects to be lazily instantiated because when a configset is
* created the {@link SolrResourceLoader} associated with it is not associated with a core
* So, we may not be able to update the core if we the schema classes are updated
* */
interface SchemaSupplier {
IndexSchema get(boolean forceFetch);
}
} }

View File

@ -81,7 +81,7 @@ public abstract class ConfigSetService {
) ? false: true; ) ? false: true;
SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted); SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted);
IndexSchema schema = createIndexSchema(dcore, solrConfig); ConfigSet.SchemaSupplier schema = force -> createIndexSchema(dcore, solrConfig, force);
return new ConfigSet(configSetName(dcore), solrConfig, schema, properties, trusted); return new ConfigSet(configSetName(dcore), solrConfig, schema, properties, trusted);
} catch (Exception e) { } catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@ -118,7 +118,7 @@ public abstract class ConfigSetService {
* @param solrConfig the core's SolrConfig * @param solrConfig the core's SolrConfig
* @return an IndexSchema * @return an IndexSchema
*/ */
protected IndexSchema createIndexSchema(CoreDescriptor cd, SolrConfig solrConfig) { protected IndexSchema createIndexSchema(CoreDescriptor cd, SolrConfig solrConfig, boolean forceFetch) {
// This is the schema name from the core descriptor. Sometimes users specify a custom schema file. // This is the schema name from the core descriptor. Sometimes users specify a custom schema file.
// Important: indexSchemaFactory.create wants this! // Important: indexSchemaFactory.create wants this!
String cdSchemaName = cd.getSchemaName(); String cdSchemaName = cd.getSchemaName();
@ -135,6 +135,7 @@ public abstract class ConfigSetService {
if (modVersion != null) { if (modVersion != null) {
// note: luceneMatchVersion influences the schema // note: luceneMatchVersion influences the schema
String cacheKey = configSet + "/" + guessSchemaName + "/" + modVersion + "/" + solrConfig.luceneMatchVersion; String cacheKey = configSet + "/" + guessSchemaName + "/" + modVersion + "/" + solrConfig.luceneMatchVersion;
if(forceFetch) schemaCache.invalidate(cacheKey);
return schemaCache.get(cacheKey, return schemaCache.get(cacheKey,
(key) -> indexSchemaFactory.create(cdSchemaName, solrConfig)); (key) -> indexSchemaFactory.create(cdSchemaName, solrConfig));
} else { } else {
@ -207,7 +208,8 @@ public abstract class ConfigSetService {
@Override @Override
public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) { public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) {
Path instanceDir = locateInstanceDir(cd); Path instanceDir = locateInstanceDir(cd);
return new SolrResourceLoader(instanceDir, parentLoader.getClassLoader()); SolrResourceLoader solrResourceLoader = new SolrResourceLoader(instanceDir, parentLoader.getClassLoader());
return solrResourceLoader;
} }
@Override @Override

View File

@ -37,6 +37,7 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
@ -1588,20 +1589,33 @@ public class CoreContainer {
return ret; return ret;
} }
/**
* reloads a core
* refer {@link CoreContainer#reload(String, UUID)} for details
*/
public void reload(String name) {
reload(name, null);
}
/** /**
* Recreates a SolrCore. * Recreates a SolrCore.
* While the new core is loading, requests will continue to be dispatched to * While the new core is loading, requests will continue to be dispatched to
* and processed by the old core * and processed by the old core
* *
* @param name the name of the SolrCore to reload * @param name the name of the SolrCore to reload
* @param coreId The unique Id of the core {@link SolrCore#uniqueId}. If this is null, it's reloaded anyway. If the current
* core has a different id, this is a no-op
*/ */
public void reload(String name) { public void reload(String name, UUID coreId) {
if (isShutDown) { if (isShutDown) {
throw new AlreadyClosedException(); throw new AlreadyClosedException();
} }
SolrCore newCore = null; SolrCore newCore = null;
SolrCore core = solrCores.getCoreFromAnyList(name, false); SolrCore core = solrCores.getCoreFromAnyList(name, false);
if (core != null) { if (core != null) {
if(coreId != null && core.uniqueId != coreId) {
//trying to reload an already unloaded core
return;
}
// The underlying core properties files may have changed, we don't really know. So we have a (perhaps) stale // The underlying core properties files may have changed, we don't really know. So we have a (perhaps) stale
// CoreDescriptor and we need to reload it from the disk files // CoreDescriptor and we need to reload it from the disk files

View File

@ -368,6 +368,10 @@ public class PluginBag<T> implements AutoCloseable {
protected final PluginInfo pluginInfo; protected final PluginInfo pluginInfo;
boolean registerAPI = false; boolean registerAPI = false;
public PluginHolder(T inst, SolrConfig.SolrPluginInfo info) {
this.inst = inst;
pluginInfo = new PluginInfo(info.tag, Collections.singletonMap("class", inst.getClass().getName()));
}
public PluginHolder(PluginInfo info) { public PluginHolder(PluginInfo info) {
this.pluginInfo = info; this.pluginInfo = info;
} }
@ -413,6 +417,10 @@ public class PluginBag<T> implements AutoCloseable {
public PluginInfo getPluginInfo() { public PluginInfo getPluginInfo() {
return pluginInfo; return pluginInfo;
} }
public String toString() {
return String.valueOf(inst);
}
} }
/** /**

View File

@ -25,7 +25,6 @@ import java.util.Map;
import org.apache.solr.common.MapSerializable; import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Pair;
import org.apache.solr.util.DOMUtil; import org.apache.solr.util.DOMUtil;
import org.w3c.dom.Node; import org.w3c.dom.Node;
import org.w3c.dom.NodeList; import org.w3c.dom.NodeList;
@ -42,6 +41,7 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME;
*/ */
public class PluginInfo implements MapSerializable { public class PluginInfo implements MapSerializable {
public final String name, className, type, pkgName; public final String name, className, type, pkgName;
public final ClassName cName;
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
public final NamedList initArgs; public final NamedList initArgs;
public final Map<String, String> attributes; public final Map<String, String> attributes;
@ -53,9 +53,9 @@ public class PluginInfo implements MapSerializable {
public PluginInfo(String type, Map<String, String> attrs, @SuppressWarnings({"rawtypes"})NamedList initArgs, List<PluginInfo> children) { public PluginInfo(String type, Map<String, String> attrs, @SuppressWarnings({"rawtypes"})NamedList initArgs, List<PluginInfo> children) {
this.type = type; this.type = type;
this.name = attrs.get(NAME); this.name = attrs.get(NAME);
Pair<String, String> parsed = parseClassName(attrs.get(CLASS_NAME)); cName = parseClassName(attrs.get(CLASS_NAME));
this.className = parsed.second(); this.className = cName.className;
this.pkgName = parsed.first(); this.pkgName = cName.pkg;
this.initArgs = initArgs; this.initArgs = initArgs;
attributes = unmodifiableMap(attrs); attributes = unmodifiableMap(attrs);
this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children); this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children);
@ -66,27 +66,45 @@ public class PluginInfo implements MapSerializable {
* This checks if it is a package name prefixed classname. * This checks if it is a package name prefixed classname.
* the return value has first = package name and second = class name * the return value has first = package name and second = class name
*/ */
public static Pair<String,String > parseClassName(String name) { public static ClassName parseClassName(String name) {
String pkgName = null; return new ClassName(name);
String className = name; }
if (name != null) {
public static class ClassName {
public final String pkg;
public final String className;
public final String original;
public ClassName(String name) {
this.original = name;
if (name == null) {
pkg = null;
className = null;
return;
}
int colonIdx = name.indexOf(':'); int colonIdx = name.indexOf(':');
if (colonIdx > -1) { if (colonIdx > -1) {
pkgName = name.substring(0, colonIdx); pkg = name.substring(0, colonIdx);
className = name.substring(colonIdx + 1); className = name.substring(colonIdx + 1);
} else {
pkg = null;
className = name;
} }
} }
return new Pair<>(pkgName, className);
@Override
public String toString() {
return original;
}
} }
public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) { public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) {
type = node.getNodeName(); type = node.getNodeName();
name = DOMUtil.getAttr(node, NAME, requireName ? err : null); name = DOMUtil.getAttr(node, NAME, requireName ? err : null);
Pair<String, String> parsed = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null)); cName = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null));
className = parsed.second(); className = cName.className;
pkgName = parsed.first(); pkgName = cName.pkg;
initArgs = DOMUtil.childNodesToNamedList(node); initArgs = DOMUtil.childNodesToNamedList(node);
attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes())); attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes()));
children = loadSubPlugins(node); children = loadSubPlugins(node);
@ -117,9 +135,9 @@ public class PluginInfo implements MapSerializable {
} }
this.type = type; this.type = type;
this.name = (String) m.get(NAME); this.name = (String) m.get(NAME);
Pair<String, String> parsed = parseClassName((String) m.get(CLASS_NAME)); cName = parseClassName((String) m.get(CLASS_NAME));
this.className = parsed.second(); this.className = cName.className;
this.pkgName = parsed.first(); this.pkgName = cName.pkg;
attributes = unmodifiableMap(m); attributes = unmodifiableMap(m);
this.children = Collections.<PluginInfo>emptyList(); this.children = Collections.<PluginInfo>emptyList();
isFromSolrConfig = true; isFromSolrConfig = true;

View File

@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
/** A generic interface to load plugin classes */
public interface SolrClassLoader {
<T> T newInstance(String cname, Class<T> expectedType, String... subpackages);
@SuppressWarnings({"rawtypes"})
<T> T newInstance(String cName, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args);
<T> Class<? extends T> findClass(String cname, Class<T> expectedType);
}

View File

@ -57,6 +57,8 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.IOUtils;
import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.pkg.PackageListeners;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.transform.TransformerFactory; import org.apache.solr.response.transform.TransformerFactory;
@ -106,6 +108,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
public static final String DEFAULT_CONF_FILE = "solrconfig.xml"; public static final String DEFAULT_CONF_FILE = "solrconfig.xml";
private RequestParams requestParams; private RequestParams requestParams;
public enum PluginOpts { public enum PluginOpts {
@ -971,6 +974,21 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
return requestParams; return requestParams;
} }
/**
* The version of package that should be loaded for a given package name
* This information is stored in the params.json in the same configset
* If params.json is absent or there is no corresponding version specified for a given package,
* this returns a null and the latest is used by the caller
*/
public String maxPackageVersion(String pkg) {
RequestParams.ParamSet p = getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS);
if (p == null) {
return null;
}
Object o = p.get().get(pkg);
if (o == null || PackageLoader.LATEST.equals(o)) return null;
return o.toString();
}
public RequestParams refreshRequestParams() { public RequestParams refreshRequestParams() {
requestParams = RequestParams.getFreshRequestParams(getResourceLoader(), requestParams); requestParams = RequestParams.getFreshRequestParams(getResourceLoader(), requestParams);

View File

@ -47,6 +47,7 @@ import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
@ -110,8 +111,7 @@ import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrCoreMetricManager; import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.pkg.PackageListeners; import org.apache.solr.pkg.*;
import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.BinaryResponseWriter; import org.apache.solr.response.BinaryResponseWriter;
@ -191,6 +191,11 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private String name; private String name;
private String logid; // used to show what name is set private String logid; // used to show what name is set
/**
* A unique id to differentiate multiple instances of the same core
* If we reload a core, the name remains same , but the id will be new
*/
public final UUID uniqueId = UUID.randomUUID();
private boolean isReloaded = false; private boolean isReloaded = false;
@ -219,6 +224,8 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private IndexReaderFactory indexReaderFactory; private IndexReaderFactory indexReaderFactory;
private final Codec codec; private final Codec codec;
private final MemClassLoader memClassLoader; private final MemClassLoader memClassLoader;
//singleton listener for all packages used in schema
private final PackageListeningClassLoader schemaPluginsLoader;
private final CircuitBreakerManager circuitBreakerManager; private final CircuitBreakerManager circuitBreakerManager;
@ -271,6 +278,9 @@ public final class SolrCore implements SolrInfoBean, Closeable {
public PackageListeners getPackageListeners() { public PackageListeners getPackageListeners() {
return packageListeners; return packageListeners;
} }
public PackageListeningClassLoader getSchemaPluginsLoader() {
return schemaPluginsLoader;
}
static int boolean_query_max_clause_count = Integer.MIN_VALUE; static int boolean_query_max_clause_count = Integer.MIN_VALUE;
@ -885,6 +895,10 @@ public final class SolrCore implements SolrInfoBean, Closeable {
public <T extends Object> T createInitInstance(PluginInfo info, Class<T> cast, String msg, String defClassName) { public <T extends Object> T createInitInstance(PluginInfo info, Class<T> cast, String msg, String defClassName) {
if (info == null) return null; if (info == null) return null;
T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName)); T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName));
return initPlugin(info, o);
}
public static <T extends Object> T initPlugin(PluginInfo info, T o) {
if (o instanceof PluginInfoInitialized) { if (o instanceof PluginInfoInitialized) {
((PluginInfoInitialized) o).init(info); ((PluginInfoInitialized) o).init(info);
} else if (o instanceof NamedListInitializedPlugin) { } else if (o instanceof NamedListInitializedPlugin) {
@ -929,15 +943,20 @@ public final class SolrCore implements SolrInfoBean, Closeable {
final CountDownLatch latch = new CountDownLatch(1); final CountDownLatch latch = new CountDownLatch(1);
try { try {
IndexSchema schema = configSet.getIndexSchema();
CoreDescriptor cd = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null"); CoreDescriptor cd = Objects.requireNonNull(coreDescriptor, "coreDescriptor cannot be null");
coreContainer.solrCores.addCoreDescriptor(cd); coreContainer.solrCores.addCoreDescriptor(cd);
setName(name); setName(name);
this.solrConfig = configSet.getSolrConfig(); this.solrConfig = configSet.getSolrConfig();
this.resourceLoader = configSet.getSolrConfig().getResourceLoader(); this.resourceLoader = configSet.getSolrConfig().getResourceLoader();
this.resourceLoader.core = this;
schemaPluginsLoader = new PackageListeningClassLoader(coreContainer, resourceLoader,
solrConfig::maxPackageVersion,
() -> setLatestSchema(configSet.getIndexSchema(true)));
this.packageListeners.addListener(schemaPluginsLoader);
IndexSchema schema = configSet.getIndexSchema();
this.configSetProperties = configSet.getProperties(); this.configSetProperties = configSet.getProperties();
// Initialize the metrics manager // Initialize the metrics manager
this.coreMetricManager = initCoreMetricManager(solrConfig); this.coreMetricManager = initCoreMetricManager(solrConfig);

View File

@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.util.*; import org.apache.lucene.analysis.util.*;
import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.Codec;
@ -59,7 +60,7 @@ import org.slf4j.LoggerFactory;
/** /**
* @since solr 1.3 * @since solr 1.3
*/ */
public class SolrResourceLoader implements ResourceLoader, Closeable { public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassLoader {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String base = "org.apache.solr"; private static final String base = "org.apache.solr";
@ -76,6 +77,12 @@ public class SolrResourceLoader implements ResourceLoader, Closeable {
protected URLClassLoader classLoader; protected URLClassLoader classLoader;
private final Path instanceDir; private final Path instanceDir;
/**
* this is set by the {@link SolrCore}
* This could be null if the core is not yet initialized
*/
SolrCore core;
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>()); private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>()); private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>()); private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
@ -198,6 +205,11 @@ public class SolrResourceLoader implements ResourceLoader, Closeable {
TokenizerFactory.reloadTokenizers(this.classLoader); TokenizerFactory.reloadTokenizers(this.classLoader);
} }
public SolrCore getCore(){
return core;
}
private static URLClassLoader addURLsToClassLoader(final URLClassLoader oldLoader, List<URL> urls) { private static URLClassLoader addURLsToClassLoader(final URLClassLoader oldLoader, List<URL> urls) {
if (urls.size() == 0) { if (urls.size() == 0) {
return oldLoader; return oldLoader;

View File

@ -18,23 +18,21 @@ package org.apache.solr.handler;
import java.io.IOException; import java.io.IOException;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.util.Arrays; import java.util.*;
import java.util.Collection; import java.util.function.BiConsumer;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.solr.api.Api; import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag; import org.apache.solr.api.ApiBag;
import org.apache.solr.cloud.ZkSolrResourceLoader; import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.Utils;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestHandler;
@ -181,6 +179,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
SimpleOrderedMap simpleOrderedMap = (SimpleOrderedMap) obj; SimpleOrderedMap simpleOrderedMap = (SimpleOrderedMap) obj;
if(name.equals(simpleOrderedMap.get("name"))) { if(name.equals(simpleOrderedMap.get("name"))) {
rsp.add(fieldName.substring(0, realName.length() - 1), simpleOrderedMap); rsp.add(fieldName.substring(0, realName.length() - 1), simpleOrderedMap);
insertPackageInfo(rsp.getValues(), req);
return; return;
} }
} }
@ -190,6 +189,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
} else { } else {
rsp.add(fieldName, o); rsp.add(fieldName, o);
} }
insertPackageInfo(rsp.getValues(), req);
return; return;
} }
@ -202,6 +202,38 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
} }
} }
/**
* If a plugin is loaded from a package, the version of the package being used should be added
* to the response
*/
@SuppressWarnings("rawtypes")
private void insertPackageInfo(Object o, SolrQueryRequest req) {
if (!req.getParams().getBool("meta", false)) return;
if (o instanceof List) {
List l = (List) o;
for (Object o1 : l) {
if (o1 instanceof NamedList || o1 instanceof List) insertPackageInfo(o1, req);
}
} else if (o instanceof NamedList) {
NamedList nl = (NamedList) o;
nl.forEach((BiConsumer) (n, v) -> {
if (v instanceof NamedList || v instanceof List) insertPackageInfo(v, req);
});
Object v = nl.get("class");
if (v instanceof String) {
String klas = (String) v;
PluginInfo.ClassName parsedClassName = new PluginInfo.ClassName(klas);
if (parsedClassName.pkg != null) {
MapWriter mw = req.getCore().getSchemaPluginsLoader().getPackageVersion(parsedClassName);
if (mw != null) nl.add("_packageinfo_", mw);
}
}
}
}
private static Set<String> subPaths = new HashSet<>(Arrays.asList( private static Set<String> subPaths = new HashSet<>(Arrays.asList(
"version", "version",
"uniquekey", "uniquekey",

View File

@ -267,7 +267,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
if (o instanceof Map) { if (o instanceof Map) {
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
Map m1 = (Map) o; Map m1 = (Map) o;
m1.put("_packageinfo_", listener.getPackageVersion()); m1.put("_packageinfo_", listener.getPackageVersion(info.cName));
} }
} }
} }

View File

@ -158,8 +158,8 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
} }
@Override @Override
protected void initNewInstance(PackageLoader.Package.Version newest) { protected Object initNewInstance(PackageLoader.Package.Version newest) {
clazz = newest.getLoader().findClass(pluginInfo.className, Expressible.class); return clazz = newest.getLoader().findClass(pluginInfo.className, Expressible.class);
} }
} }

View File

@ -166,16 +166,11 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware,
} }
@Override @Override
public void changed(PackageLoader.Package pkg) { public void changed(PackageLoader.Package pkg, Ctx ctx) {
//we could optimize this by listening to only relevant packages, //we could optimize this by listening to only relevant packages,
// but it is not worth optimizing as these are lightweight objects // but it is not worth optimizing as these are lightweight objects
components = null; components = null;
} }
@Override
public PackageLoader.Package.Version getPackageVersion() {
return null;
}
}); });
} }

View File

@ -46,7 +46,7 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.packagemanager.SolrPackage.Command; import org.apache.solr.packagemanager.SolrPackage.Command;
import org.apache.solr.packagemanager.SolrPackage.Manifest; import org.apache.solr.packagemanager.SolrPackage.Manifest;
import org.apache.solr.packagemanager.SolrPackage.Plugin; import org.apache.solr.packagemanager.SolrPackage.Plugin;
import org.apache.solr.pkg.PackagePluginHolder; import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.util.SolrCLI; import org.apache.solr.util.SolrCLI;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -250,7 +250,7 @@ public class PackageManager implements Closeable {
// Set the package version in the collection's parameters // Set the package version in the collection's parameters
try { try {
SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
"{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}"); "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackageLoader.LATEST: packageInstance.version)+"'}}}");
} catch (Exception ex) { } catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, ex); throw new SolrException(ErrorCode.SERVER_ERROR, ex);
} }
@ -296,7 +296,7 @@ public class PackageManager implements Closeable {
// Set the package version in the collection's parameters // Set the package version in the collection's parameters
try { try {
SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection),
"{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}"); "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackageLoader.LATEST: packageInstance.version) + "'}}}");
} catch (Exception ex) { } catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, ex); throw new SolrException(ErrorCode.SERVER_ERROR, ex);
} }
@ -550,7 +550,7 @@ public class PackageManager implements Closeable {
} }
} }
} }
if (version == null || version.equalsIgnoreCase(PackageUtils.LATEST) || version.equalsIgnoreCase(PackagePluginHolder.LATEST)) { if (version == null || version.equalsIgnoreCase(PackageUtils.LATEST) || version.equalsIgnoreCase(PackageLoader.LATEST)) {
return latest; return latest;
} else return null; } else return null;
} }
@ -671,7 +671,7 @@ public class PackageManager implements Closeable {
/** /**
* Given a package, return a map of collections where this package is * Given a package, return a map of collections where this package is
* installed to the installed version (which can be {@link PackagePluginHolder#LATEST}) * installed to the installed version (which can be {@link PackageLoader#LATEST})
*/ */
public Map<String, String> getDeployedCollections(String packageName) { public Map<String, String> getDeployedCollections(String packageName) {
List<String> allCollections; List<String> allCollections;

View File

@ -52,7 +52,7 @@ import org.apache.solr.filestore.PackageStoreAPI;
import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.Artifact;
import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;
import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageAPI;
import org.apache.solr.pkg.PackagePluginHolder; import org.apache.solr.pkg.PackageLoader;
import org.apache.solr.util.SolrCLI; import org.apache.solr.util.SolrCLI;
import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
@ -300,7 +300,7 @@ public class RepositoryManager {
/** /**
* Install a version of the package. Also, run verify commands in case some * Install a version of the package. Also, run verify commands in case some
* collection was using {@link PackagePluginHolder#LATEST} version of this package and got auto-updated. * collection was using {@link PackageLoader#LATEST} version of this package and got auto-updated.
*/ */
public boolean install(String packageName, String version) throws SolrException { public boolean install(String packageName, String version) throws SolrException {
SolrPackageRelease pkg = getLastPackageRelease(packageName); SolrPackageRelease pkg = getLastPackageRelease(packageName);
@ -312,7 +312,7 @@ public class RepositoryManager {
Map<String, String> collectionsDeployedIn = packageManager.getDeployedCollections(packageName); Map<String, String> collectionsDeployedIn = packageManager.getDeployedCollections(packageName);
List<String> collectionsPeggedToLatest = collectionsDeployedIn.keySet().stream(). List<String> collectionsPeggedToLatest = collectionsDeployedIn.keySet().stream().
filter(collection -> collectionsDeployedIn.get(collection).equals(PackagePluginHolder.LATEST)).collect(Collectors.toList()); filter(collection -> collectionsDeployedIn.get(collection).equals(PackageLoader.LATEST)).collect(Collectors.toList());
if (!collectionsPeggedToLatest.isEmpty()) { if (!collectionsPeggedToLatest.isEmpty()) {
PackageUtils.printGreen("Collections that will be affected (since they are configured to use $LATEST): "+collectionsPeggedToLatest); PackageUtils.printGreen("Collections that will be affected (since they are configured to use $LATEST): "+collectionsPeggedToLatest);
} }

View File

@ -192,7 +192,7 @@ public class PackageAPI {
public PkgVersion(Package.AddVersion addVersion) { public PkgVersion(Package.AddVersion addVersion) {
this.version = addVersion.version; this.version = addVersion.version;
this.files = addVersion.files; this.files = addVersion.files == null? null : Collections.unmodifiableList(addVersion.files);
this.manifest = addVersion.manifest; this.manifest = addVersion.manifest;
this.manifestSHA512 = addVersion.manifestSHA512; this.manifestSHA512 = addVersion.manifestSHA512;
} }
@ -221,6 +221,15 @@ public class PackageAPI {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
public PkgVersion copy() {
PkgVersion result = new PkgVersion();
result.version = this.version;
result.files = this.files;
result.manifest = this.manifest;
result.manifestSHA512 = this.manifestSHA512;
return result;
}
} }

View File

@ -20,10 +20,11 @@ package org.apache.solr.pkg;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.lang.ref.Reference; import java.lang.ref.Reference;
import java.lang.ref.SoftReference; import java.lang.ref.SoftReference;
import java.util.ArrayList; import java.util.*;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import org.apache.solr.common.MapWriter;
import org.apache.solr.core.PluginInfo; import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.logging.MDCLoggingContext;
@ -34,7 +35,7 @@ public class PackageListeners {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String PACKAGE_VERSIONS = "PKG_VERSIONS"; public static final String PACKAGE_VERSIONS = "PKG_VERSIONS";
private SolrCore core; private final SolrCore core;
public PackageListeners(SolrCore core) { public PackageListeners(SolrCore core) {
this.core = core; this.core = core;
@ -64,21 +65,23 @@ public class PackageListeners {
synchronized void packagesUpdated(List<PackageLoader.Package> pkgs) { synchronized void packagesUpdated(List<PackageLoader.Package> pkgs) {
MDCLoggingContext.setCore(core); MDCLoggingContext.setCore(core);
Listener.Ctx ctx = new Listener.Ctx();
try { try {
for (PackageLoader.Package pkgInfo : pkgs) { for (PackageLoader.Package pkgInfo : pkgs) {
invokeListeners(pkgInfo); invokeListeners(pkgInfo, ctx);
} }
} finally { } finally {
ctx.runLaterTasks(core::runAsync);
MDCLoggingContext.clear(); MDCLoggingContext.clear();
} }
} }
private synchronized void invokeListeners(PackageLoader.Package pkg) { private synchronized void invokeListeners(PackageLoader.Package pkg, Listener.Ctx ctx) {
for (Reference<Listener> ref : listeners) { for (Reference<Listener> ref : listeners) {
Listener listener = ref.get(); Listener listener = ref.get();
if(listener == null) continue; if(listener == null) continue;
if (listener.packageName() == null || listener.packageName().equals(pkg.name())) { if (listener.packageName() == null || listener.packageName().equals(pkg.name())) {
listener.changed(pkg); listener.changed(pkg, ctx);
} }
} }
} }
@ -96,15 +99,41 @@ public class PackageListeners {
public interface Listener { public interface Listener {
/**Name of the package or null to loisten to all package changes /**Name of the package or null to listen to all package changes */
*/
String packageName(); String packageName();
PluginInfo pluginInfo(); PluginInfo pluginInfo();
void changed(PackageLoader.Package pkg); /**A callback when the package is updated */
void changed(PackageLoader.Package pkg, Ctx ctx);
PackageLoader.Package.Version getPackageVersion(); default MapWriter getPackageVersion(PluginInfo.ClassName cName) {
return null;
}
class Ctx {
private Map<String, Runnable> runLater;
/**
* If there are multiple packages to be updated and there are multiple listeners,
* This is executed after all of the {@link Listener#changed(PackageLoader.Package, Ctx)}
* calls are invoked. The name is a unique identifier that can be used by consumers to avoid duplicate
* If no deduplication is required, use null as the name
*/
public void runLater(String name, Runnable runnable) {
if (runLater == null) runLater = new LinkedHashMap<>();
if (name == null) {
name = runnable.getClass().getSimpleName() + "@" + runnable.hashCode();
}
runLater.put(name, runnable);
}
private void runLaterTasks(Consumer<Runnable> runnableExecutor) {
if (runLater == null) return;
for (Runnable r : runLater.values()) {
runnableExecutor.accept(r);
}
}
}
} }
} }

View File

@ -0,0 +1,151 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.pkg;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrClassLoader;
import org.apache.solr.core.SolrResourceLoader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
/**
* A {@link SolrClassLoader} that is designed to listen to a set of packages.
* This class registers a listener for each package that is loaded through this.
* If any of those packages are updated, the onReload runnable is run
* */
public class PackageListeningClassLoader implements SolrClassLoader , PackageListeners.Listener {
private final CoreContainer coreContainer;
private final SolrResourceLoader coreResourceLoader;
private final Function<String, String> pkgVersionSupplier;
/** package name and the versions that we are tracking
*/
private Map<String ,PackageAPI.PkgVersion> packageVersions = new HashMap<>(1);
private final Runnable onReload;
public PackageListeningClassLoader(CoreContainer coreContainer,
SolrResourceLoader coreResourceLoader,
Function<String, String> pkgVersionSupplier,
Runnable onReload) {
this.coreContainer = coreContainer;
this.coreResourceLoader = coreResourceLoader;
this.pkgVersionSupplier = pkgVersionSupplier;
this.onReload = () -> {
packageVersions = new HashMap<>();
onReload.run();
};
}
@Override
public <T> T newInstance(String cname, Class<T> expectedType, String... subpackages) {
PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
if(cName.pkg == null){
return coreResourceLoader.newInstance(cname, expectedType, subpackages);
} else {
PackageLoader.Package.Version version = findPkgVersion(cName);
return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subpackages));
}
}
private PackageLoader.Package.Version findPkgVersion(PluginInfo.ClassName cName) {
PackageLoader.Package.Version theVersion = coreContainer.getPackageLoader().getPackage(cName.pkg).getLatest(pkgVersionSupplier.apply(cName.pkg));
packageVersions.put(cName.pkg, theVersion.getPkgVersion());
return theVersion;
}
@Override
public MapWriter getPackageVersion(PluginInfo.ClassName cName) {
if (cName.pkg == null) return null;
PackageAPI.PkgVersion p = packageVersions.get(cName.pkg);
return p == null ? null : p::writeMap;
}
private <T> T applyResourceLoaderAware(PackageLoader.Package.Version version, T obj) {
if (obj instanceof ResourceLoaderAware) {
SolrResourceLoader.assertAwareCompatibility(ResourceLoaderAware.class, obj);
try {
((ResourceLoaderAware) obj).inform(version.getLoader());
return obj;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
return obj;
}
@Override
@SuppressWarnings({"rawtypes"})
public <T> T newInstance(String cname, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args) {
PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
if (cName.pkg == null) {
return coreResourceLoader.newInstance(cname, expectedType, subPackages, params, args);
} else {
PackageLoader.Package.Version version = findPkgVersion(cName);
return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args));
}
}
@Override
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
if (cName.pkg == null) {
return coreResourceLoader.findClass(cname, expectedType);
} else {
PackageLoader.Package.Version version = findPkgVersion(cName);
return version.getLoader().findClass(cName.className, expectedType);
}
}
@Override
public String packageName() {
return null;
}
@Override
public PluginInfo pluginInfo() {
return null;
}
@Override
public void changed(PackageLoader.Package pkg, Ctx ctx) {
PackageAPI.PkgVersion currVer = packageVersions.get(pkg.name);
if (currVer == null) {
//not watching this
return;
}
String latestSupportedVersion = pkgVersionSupplier.apply(pkg.name);
if (latestSupportedVersion == null) {
//no specific version configured. use the latest
latestSupportedVersion = pkg.getLatest().getVersion();
}
if (Objects.equals(currVer.version, latestSupportedVersion)) {
//no need to update
return;
}
ctx.runLater(null, onReload);
}
}

View File

@ -50,6 +50,8 @@ import static org.apache.lucene.util.IOUtils.closeWhileHandlingException;
*/ */
public class PackageLoader implements Closeable { public class PackageLoader implements Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String LATEST = "$LATEST";
private final CoreContainer coreContainer; private final CoreContainer coreContainer;
private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>(); private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
@ -289,6 +291,9 @@ public class PackageLoader implements Closeable {
public String getVersion() { public String getVersion() {
return version.version; return version.version;
} }
public PackageAPI.PkgVersion getPkgVersion(){
return version.copy();
}
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
public Collection getFiles() { public Collection getFiles() {

View File

@ -19,9 +19,16 @@ package org.apache.solr.pkg;
import java.io.IOException; import java.io.IOException;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import org.apache.lucene.analysis.util.ResourceLoaderAware; import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.core.*; import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.plugin.SolrCoreAware; import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -55,32 +62,29 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
} }
@Override @Override
public void changed(PackageLoader.Package pkg) { public void changed(PackageLoader.Package pkg, Ctx ctx) {
reload(pkg); reload(pkg);
} }
@Override @Override
public PackageLoader.Package.Version getPackageVersion() { public MapWriter getPackageVersion(PluginInfo.ClassName cName) {
return pkgVersion; return pkgVersion == null ? null : ew -> pkgVersion.writeMap(ew);
} }
}); });
} }
private String maxVersion() { public static <T> PluginBag.PluginHolder<T> createHolder(PluginInfo info, SolrCore core, Class<T> type, String msg) {
RequestParams.ParamSet p = core.getSolrConfig().getRequestParams().getParams(PackageListeners.PACKAGE_VERSIONS); if(info.cName.pkg == null) {
if (p == null) { return new PluginBag.PluginHolder<T>(info, core.createInitInstance(info, type,msg, null));
return null; } else {
return new PackagePluginHolder<T>(info, core, SolrConfig.classVsSolrPluginInfo.get(type.getName()));
} }
Object o = p.get().get(info.pkgName);
if (o == null || LATEST.equals(o)) return null;
return o.toString();
} }
private synchronized void reload(PackageLoader.Package pkg) { private synchronized void reload(PackageLoader.Package pkg) {
String lessThan = maxVersion(); String lessThan = core.getSolrConfig().maxPackageVersion(info.pkgName);
PackageLoader.Package.Version newest = pkg.getLatest(lessThan); PackageLoader.Package.Version newest = pkg.getLatest(lessThan);
if (newest == null) { if (newest == null) {
log.error("No latest version available for package : {}", pkg.name()); log.error("No latest version available for package : {}", pkg.name());
@ -113,7 +117,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
} }
@SuppressWarnings({"unchecked"}) @SuppressWarnings({"unchecked"})
protected void initNewInstance(PackageLoader.Package.Version newest) { protected Object initNewInstance(PackageLoader.Package.Version newest) {
Object instance = SolrCore.createInstance(pluginInfo.className, Object instance = SolrCore.createInstance(pluginInfo.className,
pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader()); pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
PluginBag.initInstance(instance, pluginInfo); PluginBag.initInstance(instance, pluginInfo);
@ -128,6 +132,7 @@ public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
log.error("error closing plugin", e); log.error("error closing plugin", e);
} }
} }
return inst;
} }
private void handleAwareCallbacks(SolrResourceLoader loader, Object instance) { private void handleAwareCallbacks(SolrResourceLoader loader, Object instance) {

View File

@ -34,8 +34,8 @@ import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.Version; import org.apache.lucene.util.Version;
import org.apache.solr.analysis.TokenizerChain; import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrClassLoader;
import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.DOMUtil; import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.plugin.AbstractPluginLoader; import org.apache.solr.util.plugin.AbstractPluginLoader;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -78,7 +78,7 @@ public final class FieldTypePluginLoader
@Override @Override
protected FieldType create( SolrResourceLoader loader, protected FieldType create( SolrClassLoader loader,
String name, String name,
String className, String className,
Node node ) throws Exception { Node node ) throws Exception {
@ -189,7 +189,7 @@ public final class FieldTypePluginLoader
// //
private Analyzer readAnalyzer(Node node) throws XPathExpressionException { private Analyzer readAnalyzer(Node node) throws XPathExpressionException {
final SolrResourceLoader loader = schema.getResourceLoader(); final SolrClassLoader loader = schema.getSolrClassLoader();
// parent node used to be passed in as "fieldtype" // parent node used to be passed in as "fieldtype"
// if (!fieldtype.hasChildNodes()) return null; // if (!fieldtype.hasChildNodes()) return null;
@ -255,7 +255,7 @@ public final class FieldTypePluginLoader
@Override @Override
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
protected CharFilterFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { protected CharFilterFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception {
final Map<String,String> params = DOMUtil.toMap(node.getAttributes()); final Map<String,String> params = DOMUtil.toMap(node.getAttributes());
String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM);
params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, CharFilterFactory.class.getSimpleName()).toString()); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, CharFilterFactory.class.getSimpleName()).toString());
@ -306,7 +306,7 @@ public final class FieldTypePluginLoader
@Override @Override
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
protected TokenizerFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { protected TokenizerFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception {
final Map<String,String> params = DOMUtil.toMap(node.getAttributes()); final Map<String,String> params = DOMUtil.toMap(node.getAttributes());
String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM);
params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenizerFactory.class.getSimpleName()).toString()); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenizerFactory.class.getSimpleName()).toString());
@ -361,7 +361,7 @@ public final class FieldTypePluginLoader
{ {
@Override @Override
@SuppressWarnings({"rawtypes"}) @SuppressWarnings({"rawtypes"})
protected TokenFilterFactory create(SolrResourceLoader loader, String name, String className, Node node) throws Exception { protected TokenFilterFactory create(SolrClassLoader loader, String name, String className, Node node) throws Exception {
final Map<String,String> params = DOMUtil.toMap(node.getAttributes()); final Map<String,String> params = DOMUtil.toMap(node.getAttributes());
String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM); String configuredVersion = params.remove(LUCENE_MATCH_VERSION_PARAM);
params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenFilterFactory.class.getSimpleName()).toString()); params.put(LUCENE_MATCH_VERSION_PARAM, parseConfiguredVersion(configuredVersion, TokenFilterFactory.class.getSimpleName()).toString());

View File

@ -62,6 +62,7 @@ import org.apache.solr.common.util.Cache;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Pair; import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrClassLoader;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.core.XmlConfigFile; import org.apache.solr.core.XmlConfigFile;
@ -135,6 +136,7 @@ public class IndexSchema {
protected final Version luceneVersion; protected final Version luceneVersion;
protected float version; protected float version;
protected final SolrResourceLoader loader; protected final SolrResourceLoader loader;
protected final SolrClassLoader solrClassLoader;
protected final Properties substitutableProperties; protected final Properties substitutableProperties;
protected Map<String,SchemaField> fields = new HashMap<>(); protected Map<String,SchemaField> fields = new HashMap<>();
@ -188,6 +190,7 @@ public class IndexSchema {
protected IndexSchema(Version luceneVersion, SolrResourceLoader loader, Properties substitutableProperties) { protected IndexSchema(Version luceneVersion, SolrResourceLoader loader, Properties substitutableProperties) {
this.luceneVersion = Objects.requireNonNull(luceneVersion); this.luceneVersion = Objects.requireNonNull(luceneVersion);
this.loader = loader; this.loader = loader;
this.solrClassLoader = loader.getCore() == null? loader: loader.getCore().getSchemaPluginsLoader();
this.substitutableProperties = substitutableProperties; this.substitutableProperties = substitutableProperties;
} }
@ -208,6 +211,10 @@ public class IndexSchema {
return resourceName; return resourceName;
} }
public SolrClassLoader getSolrClassLoader() {
return solrClassLoader;
}
/** Sets the name of the resource used to instantiate this schema. */ /** Sets the name of the resource used to instantiate this schema. */
public void setResourceName(String resourceName) { public void setResourceName(String resourceName) {
this.resourceName = resourceName; this.resourceName = resourceName;
@ -391,7 +398,7 @@ public class IndexSchema {
return sf.getType().getUninversionType(sf); return sf.getType().getUninversionType(sf);
} }
// else... // else...
// It would be nice to throw a helpful error here, with a good useful message for the user, // It would be nice to throw a helpful error here, with a good useful message for the user,
// but unfortunately, inspite of the UninvertingReader class jdoc claims that the uninversion // but unfortunately, inspite of the UninvertingReader class jdoc claims that the uninversion
// process is lazy, that doesn't mean it's lazy as of "When a caller attempts ot use doc values" // process is lazy, that doesn't mean it's lazy as of "When a caller attempts ot use doc values"
@ -405,7 +412,7 @@ public class IndexSchema {
} }
/** /**
* Writes the schema in schema.xml format to the given writer * Writes the schema in schema.xml format to the given writer
*/ */
void persist(Writer writer) throws IOException { void persist(Writer writer) throws IOException {
final SolrQueryResponse response = new SolrQueryResponse(); final SolrQueryResponse response = new SolrQueryResponse();
@ -499,18 +506,18 @@ public class IndexSchema {
final FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, fieldTypes, schemaAware); final FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, fieldTypes, schemaAware);
expression = getFieldTypeXPathExpressions(); expression = getFieldTypeXPathExpressions();
NodeList nodes = (NodeList) xpath.evaluate(expression, document, XPathConstants.NODESET); NodeList nodes = (NodeList) xpath.evaluate(expression, document, XPathConstants.NODESET);
typeLoader.load(loader, nodes); typeLoader.load(solrClassLoader, nodes);
// load the fields // load the fields
Map<String,Boolean> explicitRequiredProp = loadFields(document, xpath); Map<String,Boolean> explicitRequiredProp = loadFields(document, xpath);
expression = stepsToPath(SCHEMA, SIMILARITY); // /schema/similarity expression = stepsToPath(SCHEMA, SIMILARITY); // /schema/similarity
Node node = (Node) xpath.evaluate(expression, document, XPathConstants.NODE); Node node = (Node) xpath.evaluate(expression, document, XPathConstants.NODE);
similarityFactory = readSimilarity(loader, node); similarityFactory = readSimilarity(solrClassLoader, node);
if (similarityFactory == null) { if (similarityFactory == null) {
final Class<?> simClass = SchemaSimilarityFactory.class; final Class<?> simClass = SchemaSimilarityFactory.class;
// use the loader to ensure proper SolrCoreAware handling // use the loader to ensure proper SolrCoreAware handling
similarityFactory = loader.newInstance(simClass.getName(), SimilarityFactory.class); similarityFactory = solrClassLoader.newInstance(simClass.getName(), SimilarityFactory.class);
similarityFactory.init(new ModifiableSolrParams()); similarityFactory.init(new ModifiableSolrParams());
} else { } else {
isExplicitSimilarity = true; isExplicitSimilarity = true;
@ -594,12 +601,12 @@ public class IndexSchema {
uniqueKeyField.required = true; uniqueKeyField.required = true;
requiredFields.add(uniqueKeyField); requiredFields.add(uniqueKeyField);
} }
} }
/////////////// parse out copyField commands /////////////// /////////////// parse out copyField commands ///////////////
// Map<String,ArrayList<SchemaField>> cfields = new HashMap<String,ArrayList<SchemaField>>(); // Map<String,ArrayList<SchemaField>> cfields = new HashMap<String,ArrayList<SchemaField>>();
// expression = "/schema/copyField"; // expression = "/schema/copyField";
dynamicCopyFields = new DynamicCopy[] {}; dynamicCopyFields = new DynamicCopy[] {};
loadCopyFields(document, xpath); loadCopyFields(document, xpath);
@ -754,10 +761,10 @@ public class IndexSchema {
log.error(msg); log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg); throw new SolrException(ErrorCode.SERVER_ERROR, msg);
} }
registerCopyField(source, dest, maxCharsInt); registerCopyField(source, dest, maxCharsInt);
} }
for (Map.Entry<SchemaField, Integer> entry : copyFieldTargetCounts.entrySet()) { for (Map.Entry<SchemaField, Integer> entry : copyFieldTargetCounts.entrySet()) {
if (entry.getValue() > 1 && !entry.getKey().multiValued()) { if (entry.getValue() > 1 && !entry.getKey().multiValued()) {
log.warn("Field {} is not multivalued and destination for multiople {} ({})" log.warn("Field {} is not multivalued and destination for multiople {} ({})"
@ -786,7 +793,7 @@ public class IndexSchema {
} }
return false; return false;
} }
protected boolean isValidDynamicField(List<DynamicField> dFields, SchemaField f) { protected boolean isValidDynamicField(List<DynamicField> dFields, SchemaField f) {
String glob = f.getName(); String glob = f.getName();
if (f.getDefaultValue() != null) { if (f.getDefaultValue() != null) {
@ -853,7 +860,7 @@ public class IndexSchema {
* <code>inform( SolrCore core )</code> function for <code>SolrCoreAware</code> classes. * <code>inform( SolrCore core )</code> function for <code>SolrCoreAware</code> classes.
* Outside <code>inform</code>, this could potentially throw a ConcurrentModificationException * Outside <code>inform</code>, this could potentially throw a ConcurrentModificationException
* </p> * </p>
* *
* @see SolrCoreAware * @see SolrCoreAware
*/ */
public void registerCopyField(String source, String dest, int maxChars) { public void registerCopyField(String source, String dest, int maxChars) {
@ -863,10 +870,10 @@ public class IndexSchema {
DynamicField destDynamicField = null; DynamicField destDynamicField = null;
SchemaField destSchemaField = fields.get(dest); SchemaField destSchemaField = fields.get(dest);
SchemaField sourceSchemaField = fields.get(source); SchemaField sourceSchemaField = fields.get(source);
DynamicField sourceDynamicBase = null; DynamicField sourceDynamicBase = null;
DynamicField destDynamicBase = null; DynamicField destDynamicBase = null;
boolean sourceIsDynamicFieldReference = false; boolean sourceIsDynamicFieldReference = false;
boolean sourceIsExplicitFieldGlob = false; boolean sourceIsExplicitFieldGlob = false;
@ -892,7 +899,7 @@ public class IndexSchema {
} }
} }
} }
if (null == destSchemaField || (null == sourceSchemaField && ! sourceIsExplicitFieldGlob)) { if (null == destSchemaField || (null == sourceSchemaField && ! sourceIsExplicitFieldGlob)) {
// Go through dynamicFields array only once, collecting info for both source and dest fields, if needed // Go through dynamicFields array only once, collecting info for both source and dest fields, if needed
for (DynamicField dynamicField : dynamicFields) { for (DynamicField dynamicField : dynamicFields) {
@ -914,7 +921,7 @@ public class IndexSchema {
destDynamicBase = dynamicField; destDynamicBase = dynamicField;
} }
} }
if (null != destSchemaField if (null != destSchemaField
&& (null != sourceSchemaField || sourceIsDynamicFieldReference || sourceIsExplicitFieldGlob)) { && (null != sourceSchemaField || sourceIsDynamicFieldReference || sourceIsExplicitFieldGlob)) {
break; break;
} }
@ -971,7 +978,7 @@ public class IndexSchema {
copyFieldList.add(new CopyField(sourceSchemaField, destSchemaField, maxChars)); copyFieldList.add(new CopyField(sourceSchemaField, destSchemaField, maxChars));
incrementCopyFieldTargetCount(destSchemaField); incrementCopyFieldTargetCount(destSchemaField);
} }
private void incrementCopyFieldTargetCount(SchemaField dest) { private void incrementCopyFieldTargetCount(SchemaField dest) {
copyFieldTargetCounts.put(dest, copyFieldTargetCounts.containsKey(dest) ? copyFieldTargetCounts.get(dest) + 1 : 1); copyFieldTargetCounts.put(dest, copyFieldTargetCounts.containsKey(dest) ? copyFieldTargetCounts.get(dest) + 1 : 1);
} }
@ -983,7 +990,7 @@ public class IndexSchema {
dynamicCopyFields = temp; dynamicCopyFields = temp;
} }
static SimilarityFactory readSimilarity(SolrResourceLoader loader, Node node) { static SimilarityFactory readSimilarity(SolrClassLoader loader, Node node) {
if (node==null) { if (node==null) {
return null; return null;
} else { } else {
@ -1024,7 +1031,7 @@ public class IndexSchema {
else { return new NameEquals(regex); else { return new NameEquals(regex);
} }
} }
/** Returns true if the given name matches this pattern */ /** Returns true if the given name matches this pattern */
abstract boolean matches(String name); abstract boolean matches(String name);
@ -1033,7 +1040,7 @@ public class IndexSchema {
/** Returns the result of combining this pattern's fixed string component with the given replacement */ /** Returns the result of combining this pattern's fixed string component with the given replacement */
abstract String subst(String replacement); abstract String subst(String replacement);
/** Returns the length of the original regex, including the asterisk, if any. */ /** Returns the length of the original regex, including the asterisk, if any. */
public int length() { return regex.length(); } public int length() { return regex.length(); }
@ -1076,7 +1083,7 @@ public class IndexSchema {
public int compareTo(DynamicReplacement other) { public int compareTo(DynamicReplacement other) {
return other.pattern.length() - pattern.length(); return other.pattern.length() - pattern.length();
} }
/** Returns the regex used to create this instance's pattern */ /** Returns the regex used to create this instance's pattern */
public String getRegex() { public String getRegex() {
return pattern.regex; return pattern.regex;
@ -1110,7 +1117,7 @@ public class IndexSchema {
public static class DynamicCopy extends DynamicReplacement { public static class DynamicCopy extends DynamicReplacement {
private final DynamicField destination; private final DynamicField destination;
private final int maxChars; private final int maxChars;
public int getMaxChars() { return maxChars; } public int getMaxChars() { return maxChars; }
@ -1120,7 +1127,7 @@ public class IndexSchema {
final DynamicField destDynamicBase; final DynamicField destDynamicBase;
public DynamicField getDestDynamicBase() { return destDynamicBase; } public DynamicField getDestDynamicBase() { return destDynamicBase; }
DynamicCopy(String sourceRegex, DynamicField destination, int maxChars, DynamicCopy(String sourceRegex, DynamicField destination, int maxChars,
DynamicField sourceDynamicBase, DynamicField destDynamicBase) { DynamicField sourceDynamicBase, DynamicField destDynamicBase) {
super(sourceRegex); super(sourceRegex);
this.destination = destination; this.destination = destination;
@ -1144,7 +1151,7 @@ public class IndexSchema {
return destination.makeSchemaField(targetFieldName); return destination.makeSchemaField(targetFieldName);
} }
@Override @Override
public String toString() { public String toString() {
return destination.prototype.toString(); return destination.prototype.toString();
@ -1165,7 +1172,7 @@ public class IndexSchema {
} }
return null; return null;
} }
/** /**
* Does the schema explicitly define the specified field, i.e. not as a result * Does the schema explicitly define the specified field, i.e. not as a result
* of a copyField declaration? We consider it explicitly defined if it matches * of a copyField declaration? We consider it explicitly defined if it matches
@ -1198,7 +1205,7 @@ public class IndexSchema {
} }
return false; return false;
} }
/** /**
* Returns the SchemaField that should be used for the specified field name, or * Returns the SchemaField that should be used for the specified field name, or
@ -1373,10 +1380,10 @@ public class IndexSchema {
return result; return result;
} }
/** /**
* Check if a field is used as the destination of a copyField operation * Check if a field is used as the destination of a copyField operation
* *
* @since solr 1.3 * @since solr 1.3
*/ */
public boolean isCopyFieldTarget( SchemaField f ) { public boolean isCopyFieldTarget( SchemaField f ) {
@ -1529,14 +1536,14 @@ public class IndexSchema {
/** /**
* Returns a list of copyField directives, with optional details and optionally restricting to those * Returns a list of copyField directives, with optional details and optionally restricting to those
* directives that contain the requested source and/or destination field names. * directives that contain the requested source and/or destination field names.
* *
* @param showDetails If true, source and destination dynamic bases, and explicit fields matched by source globs, * @param showDetails If true, source and destination dynamic bases, and explicit fields matched by source globs,
* will be added to dynamic copyField directives where appropriate * will be added to dynamic copyField directives where appropriate
* @param requestedSourceFields If not null, output is restricted to those copyField directives * @param requestedSourceFields If not null, output is restricted to those copyField directives
* with the requested source field names * with the requested source field names
* @param requestedDestinationFields If not null, output is restricted to those copyField directives * @param requestedDestinationFields If not null, output is restricted to those copyField directives
* with the requested destination field names * with the requested destination field names
* @return a list of copyField directives * @return a list of copyField directives
*/ */
public List<SimpleOrderedMap<Object>> getCopyFieldProperties public List<SimpleOrderedMap<Object>> getCopyFieldProperties
(boolean showDetails, Set<String> requestedSourceFields, Set<String> requestedDestinationFields) { (boolean showDetails, Set<String> requestedSourceFields, Set<String> requestedDestinationFields) {
@ -1613,7 +1620,7 @@ public class IndexSchema {
* Requires synchronizing on the object returned by * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}. * {@link #getSchemaUpdateLock()}.
* *
* @param newField the SchemaField to add * @param newField the SchemaField to add
* @param persist to persist the schema or not * @param persist to persist the schema or not
* @return a new IndexSchema based on this schema with newField added * @return a new IndexSchema based on this schema with newField added
* @see #newField(String, String, Map) * @see #newField(String, String, Map)
@ -1689,7 +1696,7 @@ public class IndexSchema {
} }
/** /**
* Copies this schema, deletes the named field from the copy, creates a new field * Copies this schema, deletes the named field from the copy, creates a new field
* with the same name using the given args, then rebinds any referring copy fields * with the same name using the given args, then rebinds any referring copy fields
* to the replacement field. * to the replacement field.
* *
@ -1699,7 +1706,7 @@ public class IndexSchema {
* Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. * Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}.
* *
* @param fieldName The name of the field to be replaced * @param fieldName The name of the field to be replaced
* @param replacementFieldType The field type of the replacement field * @param replacementFieldType The field type of the replacement field
* @param replacementArgs Initialization params for the replacement field * @param replacementArgs Initialization params for the replacement field
* @return a new IndexSchema based on this schema with the named field replaced * @return a new IndexSchema based on this schema with the named field replaced
*/ */
@ -1757,7 +1764,7 @@ public class IndexSchema {
* Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. * Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}.
* *
* @param fieldNamePattern The glob for the dynamic field to be replaced * @param fieldNamePattern The glob for the dynamic field to be replaced
* @param replacementFieldType The field type of the replacement dynamic field * @param replacementFieldType The field type of the replacement dynamic field
* @param replacementArgs Initialization params for the replacement dynamic field * @param replacementArgs Initialization params for the replacement dynamic field
* @return a new IndexSchema based on this schema with the named dynamic field replaced * @return a new IndexSchema based on this schema with the named dynamic field replaced
*/ */
@ -1787,10 +1794,10 @@ public class IndexSchema {
/** /**
* Copies this schema and adds the new copy fields to the copy. * Copies this schema and adds the new copy fields to the copy.
* *
* Requires synchronizing on the object returned by * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()} * {@link #getSchemaUpdateLock()}
* *
* @param source source field name * @param source source field name
* @param destinations collection of target field names * @param destinations collection of target field names
* @param maxChars max number of characters to copy from the source to each * @param maxChars max number of characters to copy from the source to each
@ -1812,7 +1819,7 @@ public class IndexSchema {
* Requires synchronizing on the object returned by * Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}. * {@link #getSchemaUpdateLock()}.
* *
* @param copyFields Key is the name of the source field name, value is a collection of target field names. * @param copyFields Key is the name of the source field name, value is a collection of target field names.
* Each corresponding copy field directives must exist. * Each corresponding copy field directives must exist.
* @return The new Schema with the copy fields deleted * @return The new Schema with the copy fields deleted
*/ */
@ -1824,8 +1831,8 @@ public class IndexSchema {
/** /**
* Returns a SchemaField if the given fieldName does not already * Returns a SchemaField if the given fieldName does not already
* exist in this schema, and does not match any dynamic fields * exist in this schema, and does not match any dynamic fields
* in this schema. The resulting SchemaField can be used in a call * in this schema. The resulting SchemaField can be used in a call
* to {@link #addField(SchemaField)}. * to {@link #addField(SchemaField)}.
* *
@ -1842,8 +1849,8 @@ public class IndexSchema {
} }
/** /**
* Returns a SchemaField if the given dynamic field glob does not already * Returns a SchemaField if the given dynamic field glob does not already
* exist in this schema, and does not match any dynamic fields * exist in this schema, and does not match any dynamic fields
* in this schema. The resulting SchemaField can be used in a call * in this schema. The resulting SchemaField can be used in a call
* to {@link #addField(SchemaField)}. * to {@link #addField(SchemaField)}.
* *
@ -1904,15 +1911,15 @@ public class IndexSchema {
} }
/** /**
* Copies this schema, deletes the named field type from the copy, creates a new field type * Copies this schema, deletes the named field type from the copy, creates a new field type
* with the same name using the given args, rebuilds fields and dynamic fields of the given * with the same name using the given args, rebuilds fields and dynamic fields of the given
* type, then rebinds any referring copy fields to the rebuilt fields. * type, then rebinds any referring copy fields to the rebuilt fields.
* *
* <p> * <p>
* The schema will not be persisted. * The schema will not be persisted.
* <p> * <p>
* Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}. * Requires synchronizing on the object returned by {@link #getSchemaUpdateLock()}.
* *
* @param typeName The name of the field type to be replaced * @param typeName The name of the field type to be replaced
* @param replacementClassName The class name of the replacement field type * @param replacementClassName The class name of the replacement field type
* @param replacementArgs Initialization params for the replacement field type * @param replacementArgs Initialization params for the replacement field type
@ -1944,14 +1951,14 @@ public class IndexSchema {
protected String getFieldTypeXPathExpressions() { protected String getFieldTypeXPathExpressions() {
// /schema/fieldtype | /schema/fieldType | /schema/types/fieldtype | /schema/types/fieldType // /schema/fieldtype | /schema/fieldType | /schema/types/fieldtype | /schema/types/fieldType
String expression = stepsToPath(SCHEMA, FIELD_TYPE.toLowerCase(Locale.ROOT)) // backcompat(?) String expression = stepsToPath(SCHEMA, FIELD_TYPE.toLowerCase(Locale.ROOT)) // backcompat(?)
+ XPATH_OR + stepsToPath(SCHEMA, FIELD_TYPE) + XPATH_OR + stepsToPath(SCHEMA, FIELD_TYPE)
+ XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE.toLowerCase(Locale.ROOT)) + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE.toLowerCase(Locale.ROOT))
+ XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE); + XPATH_OR + stepsToPath(SCHEMA, TYPES, FIELD_TYPE);
return expression; return expression;
} }
/** /**
* Helper method that returns <code>true</code> if the {@link #ROOT_FIELD_NAME} uses the exact * Helper method that returns <code>true</code> if the {@link #ROOT_FIELD_NAME} uses the exact
* same 'type' as the {@link #getUniqueKeyField()} * same 'type' as the {@link #getUniqueKeyField()}
* *
* @lucene.internal * @lucene.internal

View File

@ -1301,7 +1301,7 @@ public final class ManagedIndexSchema extends IndexSchema {
Map<String,FieldType> newFieldTypes = new HashMap<>(); Map<String,FieldType> newFieldTypes = new HashMap<>();
List<SchemaAware> schemaAwareList = new ArrayList<>(); List<SchemaAware> schemaAwareList = new ArrayList<>();
FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, newFieldTypes, schemaAwareList); FieldTypePluginLoader typeLoader = new FieldTypePluginLoader(this, newFieldTypes, schemaAwareList);
typeLoader.loadSingle(loader, FieldTypeXmlAdapter.toNode(options)); typeLoader.loadSingle(solrClassLoader, FieldTypeXmlAdapter.toNode(options));
FieldType ft = newFieldTypes.get(typeName); FieldType ft = newFieldTypes.get(typeName);
if (!schemaAwareList.isEmpty()) if (!schemaAwareList.isEmpty())
schemaAware.addAll(schemaAwareList); schemaAware.addAll(schemaAwareList);

View File

@ -79,7 +79,7 @@ public class PreAnalyzedField extends TextField implements HasImplicitIndexAnaly
parser = new SimplePreAnalyzedParser(); parser = new SimplePreAnalyzedParser();
} else { } else {
try { try {
Class<? extends PreAnalyzedParser> implClazz = schema.getResourceLoader().findClass(implName, PreAnalyzedParser.class); Class<? extends PreAnalyzedParser> implClazz = schema.getSolrClassLoader().findClass(implName, PreAnalyzedParser.class);
Constructor<?> c = implClazz.getConstructor(new Class<?>[0]); Constructor<?> c = implClazz.getConstructor(new Class<?>[0]);
parser = (PreAnalyzedParser) c.newInstance(new Object[0]); parser = (PreAnalyzedParser) c.newInstance(new Object[0]);
} catch (Exception e) { } catch (Exception e) {

View File

@ -23,7 +23,7 @@ import java.util.Objects;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.core.SolrClassLoader;
import org.apache.solr.util.DOMUtil; import org.apache.solr.util.DOMUtil;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -86,7 +86,7 @@ public abstract class AbstractPluginLoader<T>
* @param node - the XML node defining this plugin * @param node - the XML node defining this plugin
*/ */
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected T create( SolrResourceLoader loader, String name, String className, Node node ) throws Exception protected T create(SolrClassLoader loader, String name, String className, Node node ) throws Exception
{ {
return loader.newInstance(className, pluginClassType, getDefaultPackages()); return loader.newInstance(className, pluginClassType, getDefaultPackages());
} }
@ -135,7 +135,7 @@ public abstract class AbstractPluginLoader<T>
* If a default element is defined, it will be returned from this function. * If a default element is defined, it will be returned from this function.
* *
*/ */
public T load( SolrResourceLoader loader, NodeList nodes ) public T load(SolrClassLoader loader, NodeList nodes )
{ {
List<PluginInitInfo> info = new ArrayList<>(); List<PluginInitInfo> info = new ArrayList<>();
T defaultPlugin = null; T defaultPlugin = null;
@ -225,7 +225,7 @@ public abstract class AbstractPluginLoader<T>
* The created class for the plugin will be returned from this function. * The created class for the plugin will be returned from this function.
* *
*/ */
public T loadSingle(SolrResourceLoader loader, Node node) { public T loadSingle(SolrClassLoader loader, Node node) {
List<PluginInitInfo> info = new ArrayList<>(); List<PluginInitInfo> info = new ArrayList<>();
T plugin = null; T plugin = null;

View File

@ -112,6 +112,11 @@ openssl dgst -sha512 expressible.jar.bin
3474a1414c8329c71ef5db2d3eb6e870363bdd7224a836aab561dccf5e8bcee4974ac799e72398c7e0b0c01972bab1c7454c8a4e791a8865bb676c0440627388 3474a1414c8329c71ef5db2d3eb6e870363bdd7224a836aab561dccf5e8bcee4974ac799e72398c7e0b0c01972bab1c7454c8a4e791a8865bb676c0440627388
openssl dgst -sha512 schema-plugins.jar.bin
9299d8d5f8b358b66f113a0a91a0f55db09a41419872cb56e6e4ee402645487b75b17ec515f8035352e7fdcfac9e2e861f371d1e601869859384807b19abc2fb
=============sha256============================ =============sha256============================
openssl dgst -sha256 runtimelibs.jar.bin openssl dgst -sha256 runtimelibs.jar.bin

View File

@ -210,7 +210,7 @@ public class TestCodecSupport extends SolrTestCaseJ4 {
try { try {
CoreDescriptor cd = new CoreDescriptor(newCoreName, testSolrHome.resolve(newCoreName), coreContainer); CoreDescriptor cd = new CoreDescriptor(newCoreName, testSolrHome.resolve(newCoreName), coreContainer);
c = new SolrCore(coreContainer, cd, c = new SolrCore(coreContainer, cd,
new ConfigSet("fakeConfigset", config, schema, null, true)); new ConfigSet("fakeConfigset", config, forceFetch -> schema, null, true));
assertNull(coreContainer.registerCore(cd, c, false, false)); assertNull(coreContainer.registerCore(cd, c, false, false));
h.coreName = newCoreName; h.coreName = newCoreName;
assertEquals("We are not using the correct core", "solrconfig_codec2.xml", h.getCore().getConfigResource()); assertEquals("We are not using the correct core", "solrconfig_codec2.xml", h.getCore().getConfigResource());

View File

@ -25,10 +25,7 @@ import java.util.concurrent.Callable;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware; import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.*;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient;
@ -39,6 +36,7 @@ import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.request.beans.Package; import org.apache.solr.client.solrj.request.beans.Package;
import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SolrResponseBase;
import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.cloud.SolrCloudTestCase;
@ -46,6 +44,7 @@ import org.apache.solr.common.MapWriterMap;
import org.apache.solr.common.NavigableObject; import org.apache.solr.common.NavigableObject;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.annotation.JsonProperty; import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.SolrParams;
@ -642,6 +641,139 @@ public class TestPackages extends SolrCloudTestCase {
} }
} }
@SuppressWarnings("rawtypes")
public void testSchemaPlugins() throws Exception {
String COLLECTION_NAME = "testSchemaLoadingColl";
System.setProperty("managed.schema.mutable", "true");
MiniSolrCloudCluster cluster =
configureCluster(4)
.withJettyConfig(jetty -> jetty.enableV2(true))
.addConfig("conf", configset("cloud-managed"))
.configure();
try {
String FILE1 = "/schemapkg/schema-plugins.jar";
byte[] derFile = readFile("cryptokeys/pub_key512.der");
uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster);
postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1,
"iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A==");
Package.AddVersion add = new Package.AddVersion();
add.version = "1.0";
add.pkg = "schemapkg";
add.files = Arrays.asList(new String[]{FILE1});
V2Request req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
.withPayload(Collections.singletonMap("add", add))
.build();
req.process(cluster.getSolrClient());
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/cluster/package").
withMethod(SolrRequest.METHOD.GET)
.build().process(cluster.getSolrClient()),
Utils.makeMap(
":result:packages:schemapkg[0]:version", "1.0",
":result:packages:schemapkg[0]:files[0]", FILE1
));
CollectionAdminRequest
.createCollection(COLLECTION_NAME, "conf", 2, 2)
.process(cluster.getSolrClient());
cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
String addFieldTypeAnalyzerWithClass = "{\n" +
"'add-field-type' : {" +
" 'name' : 'myNewTextFieldWithAnalyzerClass',\n" +
" 'class':'schemapkg:my.pkg.MyTextField',\n" +
" 'analyzer' : {\n" +
" 'luceneMatchVersion':'5.0.0'" ;
// + ",\n" +
// " 'class':'schemapkg:my.pkg.MyWhitespaceAnalyzer'\n";
String charFilters =
" 'charFilters' : [{\n" +
" 'class':'schemapkg:my.pkg.MyPatternReplaceCharFilterFactory',\n" +
" 'replacement':'$1$1',\n" +
" 'pattern':'([a-zA-Z])\\\\\\\\1+'\n" +
" }],\n";
String tokenizer =
" 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n";
String filters =
" 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n";
String suffix = " }\n" +
"}}";
cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") {
@Override
public RequestWriter.ContentWriter getContentWriter(String expectedType) {
return new RequestWriter.StringPayloadContentWriter(addFieldTypeAnalyzerWithClass + ',' + charFilters + tokenizer + filters + suffix, CommonParams.JSON_MIME);
}
@Override
public SolrParams getParams() {
return null;
}
@Override
public String getCollection() {
return COLLECTION_NAME;
}
@Override
public SolrResponse createResponse(SolrClient client) {
return new SolrResponseBase();
}
});
verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass",
Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"1.0",
":fieldType:analyzer:tokenizer:_packageinfo_:version","1.0",
":fieldType:_packageinfo_:version","1.0"));
add = new Package.AddVersion();
add.version = "2.0";
add.pkg = "schemapkg";
add.files = Arrays.asList(new String[]{FILE1});
req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
.withPayload(Collections.singletonMap("add", add))
.build();
req.process(cluster.getSolrClient());
TestDistribPackageStore.assertResponseValues(10,
() -> new V2Request.Builder("/cluster/package").
withMethod(SolrRequest.METHOD.GET)
.build().process(cluster.getSolrClient()),
Utils.makeMap(
":result:packages:schemapkg[0]:version", "2.0",
":result:packages:schemapkg[0]:files[0]", FILE1
));
verifySchemaComponent(cluster.getSolrClient(), COLLECTION_NAME, "/schema/fieldtypes/myNewTextFieldWithAnalyzerClass",
Utils.makeMap(":fieldType:analyzer:charFilters[0]:_packageinfo_:version" ,"2.0",
":fieldType:analyzer:tokenizer:_packageinfo_:version","2.0",
":fieldType:_packageinfo_:version","2.0"));
} finally {
cluster.shutdown();
}
}
@SuppressWarnings({"rawtypes","unchecked"})
private void verifySchemaComponent(SolrClient client, String COLLECTION_NAME, String path,
Map expected) throws Exception {
SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
WT, JAVABIN,
"meta", "true"));
GenericSolrRequest req = new GenericSolrRequest(SolrRequest.METHOD.GET,path
, params);
TestDistribPackageStore.assertResponseValues(10,
client,
req, expected);
}
public static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception { public static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception {
ByteBuffer fileContent = getFileContent(fname); ByteBuffer fileContent = getFileContent(fname);
String sha512 = DigestUtils.sha512Hex(fileContent.array()); String sha512 = DigestUtils.sha512Hex(fileContent.array());