mirror of https://github.com/apache/lucene.git
SOLR-8906: Make transient core cache pluggable
This commit is contained in:
Erick Erickson
Shalin Shekhar Mangar
parent
ed4303ccb3
commit
e9ef17468e
|
|
@ -196,6 +196,8 @@ Other Changes
|
|||
|
||||
* SOLR-9601: Redone DataImportHandler 'tika' example, removing all unused and irrelevant definitions (Alexandre Rafalovitch)
|
||||
|
||||
* SOLR-8906: Make transient core cache pluggable (Erick Erickson)
|
||||
|
||||
================== 6.5.1 ==================
|
||||
|
||||
Bug Fixes
|
||||
|
|
|
|||
|
|
@ -131,6 +131,7 @@ public class CoreContainer {
|
|||
|
||||
protected CoreAdminHandler coreAdminHandler = null;
|
||||
protected CollectionsHandler collectionsHandler = null;
|
||||
protected TransientSolrCoreCache transientSolrCoreCache = null;
|
||||
private InfoHandler infoHandler;
|
||||
protected ConfigSetsHandler configSetsHandler = null;
|
||||
|
||||
|
|
@ -145,6 +146,8 @@ public class CoreContainer {
|
|||
|
||||
private UpdateShardHandler updateShardHandler;
|
||||
|
||||
private TransientSolrCoreCacheFactory transientCoreCache;
|
||||
|
||||
private ExecutorService coreContainerWorkExecutor = ExecutorUtil.newMDCAwareCachedThreadPool(
|
||||
new DefaultSolrThreadFactory("coreContainerWorkExecutor") );
|
||||
|
||||
|
|
@ -495,7 +498,7 @@ public class CoreContainer {
|
|||
updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());
|
||||
updateShardHandler.initializeMetrics(metricManager, SolrInfoMBean.Group.node.toString(), "updateShardHandler");
|
||||
|
||||
solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);
|
||||
transientCoreCache = TransientSolrCoreCacheFactory.newInstance(loader, this);
|
||||
|
||||
logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
|
||||
|
||||
|
|
@ -541,9 +544,9 @@ public class CoreContainer {
|
|||
String registryName = SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node);
|
||||
metricManager.registerGauge(registryName, () -> solrCores.getCores().size(),
|
||||
true, "loaded", SolrInfoMBean.Category.CONTAINER.toString(), "cores");
|
||||
metricManager.registerGauge(registryName, () -> solrCores.getCoreNames().size() - solrCores.getCores().size(),
|
||||
metricManager.registerGauge(registryName, () -> solrCores.getLoadedCoreNames().size() - solrCores.getCores().size(),
|
||||
true, "lazy",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
|
||||
metricManager.registerGauge(registryName, () -> solrCores.getAllCoreNames().size() - solrCores.getCoreNames().size(),
|
||||
metricManager.registerGauge(registryName, () -> solrCores.getAllCoreNames().size() - solrCores.getLoadedCoreNames().size(),
|
||||
true, "unloaded",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
|
||||
metricManager.registerGauge(registryName, () -> cfg.getCoreRootDirectory().toFile().getTotalSpace(),
|
||||
true, "totalSpace", SolrInfoMBean.Category.CONTAINER.toString(), "fs");
|
||||
|
|
@ -635,6 +638,16 @@ public class CoreContainer {
|
|||
}
|
||||
}
|
||||
|
||||
public TransientSolrCoreCache getTransientCacheHandler() {
|
||||
|
||||
if (transientCoreCache == null) {
|
||||
log.error("No transient handler has been defined. Check solr.xml to see if an attempt to provide a custom " +
|
||||
"TransientSolrCoreCacheFactory was done incorrectly since the default should have been used otherwise.");
|
||||
return null;
|
||||
}
|
||||
return transientCoreCache.getTransientSolrCoreCache();
|
||||
}
|
||||
|
||||
public void securityNodeChanged() {
|
||||
log.info("Security node changed, reloading security.json");
|
||||
reloadSecurityProperties();
|
||||
|
|
@ -1082,10 +1095,10 @@ public class CoreContainer {
|
|||
}
|
||||
|
||||
/**
|
||||
* @return a Collection of the names that cores are mapped to
|
||||
* @return a Collection of the names that loaded cores are mapped to
|
||||
*/
|
||||
public Collection<String> getCoreNames() {
|
||||
return solrCores.getCoreNames();
|
||||
return solrCores.getLoadedCoreNames();
|
||||
}
|
||||
|
||||
/** This method is currently experimental.
|
||||
|
|
@ -1098,6 +1111,8 @@ public class CoreContainer {
|
|||
/**
|
||||
* get a list of all the cores that are currently loaded
|
||||
* @return a list of al lthe available core names in either permanent or transient core lists.
|
||||
*
|
||||
* Note: this implies that the core is loaded
|
||||
*/
|
||||
public Collection<String> getAllCoreNames() {
|
||||
return solrCores.getAllCoreNames();
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ public class NodeConfig {
|
|||
|
||||
private final Integer coreLoadThreads;
|
||||
|
||||
@Deprecated
|
||||
// This should be part of the transientCacheConfig, remove in 7.0
|
||||
private final int transientCacheSize;
|
||||
|
||||
private final boolean useSchemaCache;
|
||||
|
|
@ -62,6 +64,8 @@ public class NodeConfig {
|
|||
|
||||
private final PluginInfo[] metricReporterPlugins;
|
||||
|
||||
private final PluginInfo transientCacheConfig;
|
||||
|
||||
private NodeConfig(String nodeName, Path coreRootDirectory, Path configSetBaseDirectory, String sharedLibDirectory,
|
||||
PluginInfo shardHandlerFactoryConfig, UpdateShardHandlerConfig updateShardHandlerConfig,
|
||||
String coreAdminHandlerClass, String collectionsAdminHandlerClass,
|
||||
|
|
@ -69,7 +73,7 @@ public class NodeConfig {
|
|||
LogWatcherConfig logWatcherConfig, CloudConfig cloudConfig, Integer coreLoadThreads,
|
||||
int transientCacheSize, boolean useSchemaCache, String managementPath, SolrResourceLoader loader,
|
||||
Properties solrProperties, PluginInfo[] backupRepositoryPlugins,
|
||||
PluginInfo[] metricReporterPlugins) {
|
||||
PluginInfo[] metricReporterPlugins, PluginInfo transientCacheConfig) {
|
||||
this.nodeName = nodeName;
|
||||
this.coreRootDirectory = coreRootDirectory;
|
||||
this.configSetBaseDirectory = configSetBaseDirectory;
|
||||
|
|
@ -90,6 +94,7 @@ public class NodeConfig {
|
|||
this.solrProperties = solrProperties;
|
||||
this.backupRepositoryPlugins = backupRepositoryPlugins;
|
||||
this.metricReporterPlugins = metricReporterPlugins;
|
||||
this.transientCacheConfig = transientCacheConfig;
|
||||
|
||||
if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
|
|
@ -182,6 +187,8 @@ public class NodeConfig {
|
|||
return metricReporterPlugins;
|
||||
}
|
||||
|
||||
public PluginInfo getTransientCachePluginInfo() { return transientCacheConfig; }
|
||||
|
||||
public static class NodeConfigBuilder {
|
||||
|
||||
private Path coreRootDirectory;
|
||||
|
|
@ -195,13 +202,16 @@ public class NodeConfig {
|
|||
private String configSetsHandlerClass = DEFAULT_CONFIGSETSHANDLERCLASS;
|
||||
private LogWatcherConfig logWatcherConfig = new LogWatcherConfig(true, null, null, 50);
|
||||
private CloudConfig cloudConfig;
|
||||
private Integer coreLoadThreads;
|
||||
private int coreLoadThreads = DEFAULT_CORE_LOAD_THREADS;
|
||||
@Deprecated
|
||||
//Remove in 7.0 and put it all in the transientCache element in solrconfig.xml
|
||||
private int transientCacheSize = DEFAULT_TRANSIENT_CACHE_SIZE;
|
||||
private boolean useSchemaCache = false;
|
||||
private String managementPath;
|
||||
private Properties solrProperties = new Properties();
|
||||
private PluginInfo[] backupRepositoryPlugins;
|
||||
private PluginInfo[] metricReporterPlugins;
|
||||
private PluginInfo transientCacheConfig;
|
||||
|
||||
private final SolrResourceLoader loader;
|
||||
private final String nodeName;
|
||||
|
|
@ -210,7 +220,7 @@ public class NodeConfig {
|
|||
//No:of core load threads in cloud mode is set to a default of 8
|
||||
public static final int DEFAULT_CORE_LOAD_THREADS_IN_CLOUD = 8;
|
||||
|
||||
private static final int DEFAULT_TRANSIENT_CACHE_SIZE = Integer.MAX_VALUE;
|
||||
public static final int DEFAULT_TRANSIENT_CACHE_SIZE = Integer.MAX_VALUE;
|
||||
|
||||
private static final String DEFAULT_ADMINHANDLERCLASS = "org.apache.solr.handler.admin.CoreAdminHandler";
|
||||
private static final String DEFAULT_INFOHANDLERCLASS = "org.apache.solr.handler.admin.InfoHandler";
|
||||
|
|
@ -284,6 +294,8 @@ public class NodeConfig {
|
|||
return this;
|
||||
}
|
||||
|
||||
// Remove in Solr 7.0
|
||||
@Deprecated
|
||||
public NodeConfigBuilder setTransientCacheSize(int transientCacheSize) {
|
||||
this.transientCacheSize = transientCacheSize;
|
||||
return this;
|
||||
|
|
@ -313,12 +325,17 @@ public class NodeConfig {
|
|||
this.metricReporterPlugins = metricReporterPlugins;
|
||||
return this;
|
||||
}
|
||||
|
||||
public NodeConfigBuilder setSolrCoreCacheFactoryConfig(PluginInfo transientCacheConfig) {
|
||||
this.transientCacheConfig = transientCacheConfig;
|
||||
return this;
|
||||
}
|
||||
|
||||
public NodeConfig build() {
|
||||
return new NodeConfig(nodeName, coreRootDirectory, configSetBaseDirectory, sharedLibDirectory, shardHandlerFactoryConfig,
|
||||
updateShardHandlerConfig, coreAdminHandlerClass, collectionsAdminHandlerClass, infoHandlerClass, configSetsHandlerClass,
|
||||
logWatcherConfig, cloudConfig, coreLoadThreads, transientCacheSize, useSchemaCache, managementPath, loader, solrProperties,
|
||||
backupRepositoryPlugins, metricReporterPlugins);
|
||||
backupRepositoryPlugins, metricReporterPlugins, transientCacheConfig);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
package org.apache.solr.core;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.http.annotation.Experimental;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.util.ExecutorUtil;
|
||||
import org.apache.solr.logging.MDCLoggingContext;
|
||||
|
|
@ -32,6 +33,8 @@ import java.util.HashSet;
|
|||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Observable;
|
||||
import java.util.Observer;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
|
@ -39,15 +42,12 @@ import java.util.concurrent.ExecutorService;
|
|||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
||||
class SolrCores {
|
||||
class SolrCores implements Observer {
|
||||
|
||||
private static Object modifyLock = new Object(); // for locking around manipulating any of the core maps.
|
||||
private final Map<String, SolrCore> cores = new LinkedHashMap<>(); // For "permanent" cores
|
||||
|
||||
//WARNING! The _only_ place you put anything into the list of transient cores is with the putTransientCore method!
|
||||
private Map<String, SolrCore> transientCores = new LinkedHashMap<>(); // For "lazily loaded" cores
|
||||
|
||||
private final Map<String, CoreDescriptor> dynamicDescriptors = new LinkedHashMap<>();
|
||||
private final Map<String, CoreDescriptor> lazyDescriptors = new LinkedHashMap<>();
|
||||
|
||||
private final CoreContainer container;
|
||||
|
||||
|
|
@ -66,33 +66,19 @@ class SolrCores {
|
|||
SolrCores(CoreContainer container) {
|
||||
this.container = container;
|
||||
}
|
||||
|
||||
// Trivial helper method for load, note it implements LRU on transient cores. Also note, if
|
||||
// there is no setting for max size, nothing is done and all cores go in the regular "cores" list
|
||||
protected void allocateLazyCores(final int cacheSize, final SolrResourceLoader loader) {
|
||||
if (cacheSize != Integer.MAX_VALUE) {
|
||||
log.info("Allocating transient cache for {} transient cores", cacheSize);
|
||||
transientCores = new LinkedHashMap<String, SolrCore>(cacheSize, 0.75f, true) {
|
||||
@Override
|
||||
protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
|
||||
if (size() > cacheSize) {
|
||||
synchronized (modifyLock) {
|
||||
SolrCore coreToClose = eldest.getValue();
|
||||
log.info("Closing transient core [{}]", coreToClose.getName());
|
||||
pendingCloses.add(coreToClose); // Essentially just queue this core up for closing.
|
||||
modifyLock.notifyAll(); // Wakes up closer thread too
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
protected void putDynamicDescriptor(String rawName, CoreDescriptor p) {
|
||||
|
||||
protected void putDynamicDescriptor(String rawName, CoreDescriptor cd) {
|
||||
synchronized (modifyLock) {
|
||||
dynamicDescriptors.put(rawName, p);
|
||||
if (cd.isTransient()) {
|
||||
if (container.getTransientCacheHandler() != null) {
|
||||
container.getTransientCacheHandler().addTransientDescriptor(rawName, cd);
|
||||
} else {
|
||||
log.error("Tried to add transient core to transient handler, but no transient core handler has been found. "
|
||||
+ " Descriptor: " + cd.toString());
|
||||
}
|
||||
} else {
|
||||
lazyDescriptors.put(rawName, cd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -101,19 +87,25 @@ class SolrCores {
|
|||
protected void close() {
|
||||
Collection<SolrCore> coreList = new ArrayList<>();
|
||||
|
||||
|
||||
TransientSolrCoreCache transientSolrCoreCache = container.getTransientCacheHandler();
|
||||
// Release observer
|
||||
if (transientSolrCoreCache != null) {
|
||||
transientSolrCoreCache.close();
|
||||
}
|
||||
|
||||
// It might be possible for one of the cores to move from one list to another while we're closing them. So
|
||||
// loop through the lists until they're all empty. In particular, the core could have moved from the transient
|
||||
// list to the pendingCloses list.
|
||||
|
||||
do {
|
||||
coreList.clear();
|
||||
synchronized (modifyLock) {
|
||||
// make a copy of the cores then clear the map so the core isn't handed out to a request again
|
||||
coreList.addAll(cores.values());
|
||||
cores.clear();
|
||||
|
||||
coreList.addAll(transientCores.values());
|
||||
transientCores.clear();
|
||||
if (transientSolrCoreCache != null) {
|
||||
coreList.addAll(transientSolrCoreCache.prepareForShutdown());
|
||||
}
|
||||
|
||||
coreList.addAll(pendingCloses);
|
||||
pendingCloses.clear();
|
||||
|
|
@ -147,10 +139,12 @@ class SolrCores {
|
|||
|
||||
//WARNING! This should be the _only_ place you put anything into the list of transient cores!
|
||||
protected SolrCore putTransientCore(NodeConfig cfg, String name, SolrCore core, SolrResourceLoader loader) {
|
||||
SolrCore retCore;
|
||||
SolrCore retCore = null;
|
||||
log.info("Opening transient core {}", name);
|
||||
synchronized (modifyLock) {
|
||||
retCore = transientCores.put(name, core);
|
||||
if (container.getTransientCacheHandler() != null) {
|
||||
retCore = container.getTransientCacheHandler().addCore(name, core);
|
||||
}
|
||||
}
|
||||
return retCore;
|
||||
}
|
||||
|
|
@ -161,6 +155,17 @@ class SolrCores {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return A list of "permanent" cores, i.e. cores that may not be swapped out and are currently loaded.
|
||||
*
|
||||
* A core may be non-transient but still lazily loaded. If it is "permanent" and lazy-load _and_
|
||||
* not yet loaded it will _not_ be returned by this call.
|
||||
*
|
||||
* Note: This is one of the places where SolrCloud is incompatible with Transient Cores. This call is used in
|
||||
* cancelRecoveries, transient cores don't participate.
|
||||
*/
|
||||
|
||||
List<SolrCore> getCores() {
|
||||
List<SolrCore> lst = new ArrayList<>();
|
||||
|
||||
|
|
@ -170,16 +175,34 @@ class SolrCores {
|
|||
}
|
||||
}
|
||||
|
||||
Set<String> getCoreNames() {
|
||||
/**
|
||||
* Gets the cores that are currently loaded, i.e. cores that have
|
||||
* 1> loadOnStartup=true and are either not-transient or, if transient, have been loaded and have not been swapped out
|
||||
* 2> loadOnStartup=false and have been loaded but either non-transient or have not been swapped out.
|
||||
*
|
||||
* Put another way, this will not return any names of cores that are lazily loaded but have not been called for yet
|
||||
* or are transient and either not loaded or have been swapped out.
|
||||
*
|
||||
* @return List of currently loaded cores.
|
||||
*/
|
||||
Set<String> getLoadedCoreNames() {
|
||||
Set<String> set = new TreeSet<>();
|
||||
|
||||
synchronized (modifyLock) {
|
||||
set.addAll(cores.keySet());
|
||||
set.addAll(transientCores.keySet());
|
||||
if (container.getTransientCacheHandler() != null) {
|
||||
set.addAll(container.getTransientCacheHandler().getLoadedCoreNames());
|
||||
}
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
/** This method is currently experimental.
|
||||
* @return a Collection of the names that a specific core is mapped to.
|
||||
*
|
||||
* Note: this implies that the core is loaded
|
||||
*/
|
||||
@Experimental
|
||||
List<String> getCoreNames(SolrCore core) {
|
||||
List<String> lst = new ArrayList<>();
|
||||
|
||||
|
|
@ -189,26 +212,26 @@ class SolrCores {
|
|||
lst.add(entry.getKey());
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, SolrCore> entry : transientCores.entrySet()) {
|
||||
if (core == entry.getValue()) {
|
||||
lst.add(entry.getKey());
|
||||
}
|
||||
if (container.getTransientCacheHandler() != null) {
|
||||
lst.addAll(container.getTransientCacheHandler().getNamesForCore(core));
|
||||
}
|
||||
}
|
||||
return lst;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a list of all cores, loaded and unloaded (dynamic)
|
||||
* Gets a list of all cores, loaded and unloaded
|
||||
*
|
||||
* @return all cores names, whether loaded or unloaded.
|
||||
* @return all cores names, whether loaded or unloaded, transient or permenent.
|
||||
*/
|
||||
public Collection<String> getAllCoreNames() {
|
||||
Set<String> set = new TreeSet<>();
|
||||
synchronized (modifyLock) {
|
||||
set.addAll(cores.keySet());
|
||||
set.addAll(transientCores.keySet());
|
||||
set.addAll(dynamicDescriptors.keySet());
|
||||
if (container.getTransientCacheHandler() != null) {
|
||||
set.addAll(container.getTransientCacheHandler().getAllCoreNames());
|
||||
}
|
||||
set.addAll(lazyDescriptors.keySet());
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
|
@ -251,14 +274,15 @@ class SolrCores {
|
|||
protected SolrCore remove(String name) {
|
||||
|
||||
synchronized (modifyLock) {
|
||||
SolrCore tmp = cores.remove(name);
|
||||
SolrCore ret = null;
|
||||
ret = (ret == null) ? tmp : ret;
|
||||
SolrCore ret = cores.remove(name);
|
||||
// It could have been a newly-created core. It could have been a transient core. The newly-created cores
|
||||
// in particular should be checked. It could have been a dynamic core.
|
||||
tmp = transientCores.remove(name);
|
||||
ret = (ret == null) ? tmp : ret;
|
||||
dynamicDescriptors.remove(name);
|
||||
TransientSolrCoreCache transientHandler = container.getTransientCacheHandler();
|
||||
if (ret == null && transientHandler != null) {
|
||||
ret = transientHandler.removeCore(name);
|
||||
transientHandler.removeTransientDescriptor(name);
|
||||
}
|
||||
lazyDescriptors.remove(name);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
|
@ -268,8 +292,8 @@ class SolrCores {
|
|||
synchronized (modifyLock) {
|
||||
SolrCore core = cores.get(name);
|
||||
|
||||
if (core == null) {
|
||||
core = transientCores.get(name);
|
||||
if (core == null && container.getTransientCacheHandler() != null) {
|
||||
core = container.getTransientCacheHandler().getCore(name);
|
||||
}
|
||||
|
||||
if (core != null && incRefCount) {
|
||||
|
|
@ -282,7 +306,9 @@ class SolrCores {
|
|||
|
||||
protected CoreDescriptor getDynamicDescriptor(String name) {
|
||||
synchronized (modifyLock) {
|
||||
return dynamicDescriptors.get(name);
|
||||
CoreDescriptor cd = lazyDescriptors.get(name);
|
||||
if (cd != null || container.getTransientCacheHandler() == null) return cd;
|
||||
return container.getTransientCacheHandler().getTransientDescriptor(name);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -295,7 +321,7 @@ class SolrCores {
|
|||
if (cores.containsKey(name)) {
|
||||
return true;
|
||||
}
|
||||
if (transientCores.containsKey(name)) {
|
||||
if (container.getTransientCacheHandler() != null && container.getTransientCacheHandler().containsCore(name)) {
|
||||
// Check pending
|
||||
for (SolrCore core : pendingCloses) {
|
||||
if (core.getName().equals(name)) {
|
||||
|
|
@ -314,7 +340,7 @@ class SolrCores {
|
|||
if (cores.containsKey(name)) {
|
||||
return true;
|
||||
}
|
||||
if (transientCores.containsKey(name)) {
|
||||
if (container.getTransientCacheHandler() != null && container.getTransientCacheHandler().containsCore(name)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -324,13 +350,16 @@ class SolrCores {
|
|||
|
||||
protected CoreDescriptor getUnloadedCoreDescriptor(String cname) {
|
||||
synchronized (modifyLock) {
|
||||
CoreDescriptor desc = dynamicDescriptors.get(cname);
|
||||
CoreDescriptor desc = lazyDescriptors.get(cname);
|
||||
if (desc == null) {
|
||||
return null;
|
||||
if (container.getTransientCacheHandler() == null) return null;
|
||||
desc = container.getTransientCacheHandler().getTransientDescriptor(cname);
|
||||
if (desc == null) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return new CoreDescriptor(cname, desc);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Wait here until any pending operations (load, unload or reload) are completed on this core.
|
||||
|
|
@ -412,9 +441,9 @@ class SolrCores {
|
|||
synchronized (modifyLock) {
|
||||
if (cores.containsKey(coreName))
|
||||
return cores.get(coreName).getCoreDescriptor();
|
||||
if (dynamicDescriptors.containsKey(coreName))
|
||||
return dynamicDescriptors.get(coreName);
|
||||
return null;
|
||||
if (lazyDescriptors.containsKey(coreName) || container.getTransientCacheHandler() == null)
|
||||
return lazyDescriptors.get(coreName);
|
||||
return container.getTransientCacheHandler().getTransientDescriptor(coreName);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -494,4 +523,13 @@ class SolrCores {
|
|||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Let transient cache implementation tell us when it ages out a corel
|
||||
@Override
|
||||
public void update(Observable o, Object arg) {
|
||||
synchronized (modifyLock) {
|
||||
pendingCloses.add((SolrCore) arg); // Essentially just queue this core up for closing.
|
||||
modifyLock.notifyAll(); // Wakes up closer thread too
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ public class SolrXmlConfig {
|
|||
NodeConfig.NodeConfigBuilder configBuilder = new NodeConfig.NodeConfigBuilder(nodeName, config.getResourceLoader());
|
||||
configBuilder.setUpdateShardHandlerConfig(updateConfig);
|
||||
configBuilder.setShardHandlerFactoryConfig(getShardHandlerFactoryPluginInfo(config));
|
||||
configBuilder.setSolrCoreCacheFactoryConfig(getTransientCoreCacheFactoryPluginInfo(config));
|
||||
configBuilder.setLogWatcherConfig(loadLogWatcherConfig(config, "solr/logging/*[@name]", "solr/logging/watcher/*[@name]"));
|
||||
configBuilder.setSolrProperties(loadProperties(config));
|
||||
if (cloudConfig != null)
|
||||
|
|
@ -456,5 +457,9 @@ public class SolrXmlConfig {
|
|||
}
|
||||
return configs;
|
||||
}
|
||||
private static PluginInfo getTransientCoreCacheFactoryPluginInfo(Config config) {
|
||||
Node node = config.getNode("solr/transientCoreCacheFactory", false);
|
||||
return (node == null) ? null : new PluginInfo(node, "transientCoreCacheFactory", false, true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.core;
|
||||
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Observable;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.http.annotation.Experimental;
|
||||
|
||||
/**
|
||||
* The base class for custom transient core maintenance. Any custom plugin that want's to take control of transient
|
||||
* caches (i.e. any core defined with transient=true) should override this class.
|
||||
*
|
||||
* Register your plugin in solr.xml similarly to:
|
||||
*
|
||||
* <transientCoreCacheFactory name="transientCoreCacheFactory" class="TransientSolrCoreCacheFactoryDefault">
|
||||
* <int name="transientCacheSize">4</int>
|
||||
* </transientCoreCacheFactory>
|
||||
*
|
||||
*
|
||||
* WARNING: There is quite a bit of higher-level locking done by the CoreContainer to avoid various race conditions
|
||||
* etc. You should _only_ manipulate them within the method calls designed to change them. E.g.
|
||||
* only add to the transient core descriptors in addTransientDescriptor etc.
|
||||
*
|
||||
* Trust the higher-level code (mainly SolrCores and CoreContainer) to call the appropriate operations when
|
||||
* necessary and to coordinate shutting down cores, manipulating the internal structures and the like..
|
||||
*
|
||||
* The only real action you should _initiate_ is to close a core for whatever reason, and do that by
|
||||
* calling notifyObservers(coreToClose); The observer will call back to removeCore(name) at the appropriate
|
||||
* time. There is no need to directly remove the core _at that time_ from the transientCores list, a call
|
||||
* will come back to this class when CoreContainer is closing this core.
|
||||
*
|
||||
* CoreDescriptors are read-once. During "core discovery" all valid descriptors are enumerated and added to
|
||||
* the appropriate list. Thereafter, they are NOT re-read from disk. In those situations where you want
|
||||
* to re-define the coreDescriptor, maintain a "side list" of changed core descriptors. Then override
|
||||
* getTransientDescriptor to return your new core descriptor. NOTE: assuming you've already closed the
|
||||
* core, the _next_ time that core is required getTransientDescriptor will be called and if you return the
|
||||
* new core descriptor your re-definition should be honored. You'll have to maintain this list for the
|
||||
* duration of this Solr instance running. If you persist the coreDescriptor, then next time Solr starts
|
||||
* up the new definition will be read.
|
||||
*
|
||||
*
|
||||
* If you need to manipulate the return, for instance block a core from being loaded for some period of time, override
|
||||
* say getTransientDescriptor and return null.
|
||||
*
|
||||
* In particular, DO NOT reach into the transientCores structure from a method called to manipulate core descriptors
|
||||
* or vice-versa.
|
||||
*/
|
||||
public abstract class TransientSolrCoreCache extends Observable {
|
||||
|
||||
// Gets the core container that encloses this cache.
|
||||
public abstract CoreContainer getContainer();
|
||||
|
||||
// Add the newly-opened core to the list of open cores.
|
||||
public abstract SolrCore addCore(String name, SolrCore core);
|
||||
|
||||
// Return the names of all possible cores, whether they are currently loaded or not.
|
||||
public abstract Set<String> getAllCoreNames();
|
||||
|
||||
// Return the names of all currently loaded cores
|
||||
public abstract Set<String> getLoadedCoreNames();
|
||||
|
||||
// Remove a core from the internal structures, presumably it
|
||||
// being closed. If the core is re-opened, it will be readded by CoreContainer.
|
||||
public abstract SolrCore removeCore(String name);
|
||||
|
||||
// Get the core associated with the name. Return null if you don't want this core to be used.
|
||||
public abstract SolrCore getCore(String name);
|
||||
|
||||
// reutrn true if the cache contains the named core.
|
||||
public abstract boolean containsCore(String name);
|
||||
|
||||
// This method will be called when the container is to be shut down. It should return all
|
||||
// transient solr cores and clear any internal structures that hold them.
|
||||
public abstract Collection<SolrCore> prepareForShutdown();
|
||||
|
||||
// These methods allow the implementation to maintain control over the core descriptors.
|
||||
|
||||
// This method will only be called during core discovery at startup.
|
||||
public abstract void addTransientDescriptor(String rawName, CoreDescriptor cd);
|
||||
|
||||
// This method is used when opening cores and the like. If you want to change a core's descriptor, override this
|
||||
// method and return the current core descriptor.
|
||||
public abstract CoreDescriptor getTransientDescriptor(String name);
|
||||
|
||||
|
||||
// Remove the core descriptor from your list of transient descriptors.
|
||||
public abstract CoreDescriptor removeTransientDescriptor(String name);
|
||||
|
||||
// Find all the names a specific core is mapped to. Should not return null, return empty set instead.
|
||||
@Experimental
|
||||
public List<String> getNamesForCore(SolrCore core) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Must be called in order to free resources!
|
||||
*/
|
||||
public abstract void close();
|
||||
|
||||
|
||||
// These two methods allow custom implementations to communicate arbitrary information as necessary.
|
||||
public abstract int getStatus(String coreName);
|
||||
public abstract void setStatus(String coreName, int status);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.core;
|
||||
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Observer;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class TransientSolrCoreCacheDefault extends TransientSolrCoreCache {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
private int cacheSize = NodeConfig.NodeConfigBuilder.DEFAULT_TRANSIENT_CACHE_SIZE;
|
||||
|
||||
protected Observer observer;
|
||||
protected CoreContainer coreContainer;
|
||||
|
||||
protected final Map<String, CoreDescriptor> transientDescriptors = new LinkedHashMap<>();
|
||||
|
||||
//WARNING! The _only_ place you put anything into the list of transient cores is with the putTransientCore method!
|
||||
protected Map<String, SolrCore> transientCores = new LinkedHashMap<>(); // For "lazily loaded" cores
|
||||
|
||||
/**
|
||||
* @param container The enclosing CoreContainer. It allows us to access everything we need.
|
||||
*/
|
||||
public TransientSolrCoreCacheDefault(final CoreContainer container) {
|
||||
this.coreContainer = container;
|
||||
this.observer= coreContainer.solrCores;
|
||||
|
||||
NodeConfig cfg = container.getNodeConfig();
|
||||
if (cfg.getTransientCachePluginInfo() == null) {
|
||||
// Still handle just having transientCacheSize defined in the body of solr.xml not in a transient handler clause.
|
||||
// deprecate this for 7.0?
|
||||
this.cacheSize = cfg.getTransientCacheSize();
|
||||
} else {
|
||||
NamedList args = cfg.getTransientCachePluginInfo().initArgs;
|
||||
Object obj = args.get("transientCacheSize");
|
||||
if (obj != null) {
|
||||
this.cacheSize = (int) obj;
|
||||
}
|
||||
}
|
||||
doInit();
|
||||
}
|
||||
// This just moves the
|
||||
private void doInit() {
|
||||
NodeConfig cfg = coreContainer.getNodeConfig();
|
||||
if (cfg.getTransientCachePluginInfo() == null) {
|
||||
// Still handle just having transientCacheSize defined in the body of solr.xml not in a transient handler clause.
|
||||
this.cacheSize = cfg.getTransientCacheSize();
|
||||
} else {
|
||||
NamedList args = cfg.getTransientCachePluginInfo().initArgs;
|
||||
Object obj = args.get("transientCacheSize");
|
||||
if (obj != null) {
|
||||
this.cacheSize = (int) obj;
|
||||
}
|
||||
}
|
||||
|
||||
log.info("Allocating transient cache for {} transient cores", cacheSize);
|
||||
addObserver(this.observer);
|
||||
// it's possible for cache
|
||||
if (cacheSize < 0) { // Trap old flag
|
||||
cacheSize = Integer.MAX_VALUE;
|
||||
}
|
||||
// Now don't allow ridiculous allocations here, if the size is > 1,000, we'll just deal with
|
||||
// adding cores as they're opened. This blows up with the marker value of -1.
|
||||
transientCores = new LinkedHashMap<String, SolrCore>(Math.min(cacheSize, 1000), 0.75f, true) {
|
||||
@Override
|
||||
protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
|
||||
if (size() > cacheSize) {
|
||||
SolrCore coreToClose = eldest.getValue();
|
||||
setChanged();
|
||||
notifyObservers(coreToClose);
|
||||
log.info("Closing transient core [{}]", coreToClose.getName());
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Collection<SolrCore> prepareForShutdown() {
|
||||
// Returna copy of the values
|
||||
List<SolrCore> ret = new ArrayList(transientCores.values());
|
||||
transientCores.clear();
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CoreContainer getContainer() { return this.coreContainer; }
|
||||
|
||||
@Override
|
||||
public SolrCore addCore(String name, SolrCore core) {
|
||||
return transientCores.put(name, core);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getAllCoreNames() {
|
||||
return transientDescriptors.keySet();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getLoadedCoreNames() {
|
||||
return transientCores.keySet();
|
||||
}
|
||||
|
||||
// Remove a core from the internal structures, presumably it
|
||||
// being closed. If the core is re-opened, it will be readded by CoreContainer.
|
||||
@Override
|
||||
public SolrCore removeCore(String name) {
|
||||
return transientCores.remove(name);
|
||||
}
|
||||
|
||||
// Get the core associated with the name. Return null if you don't want this core to be used.
|
||||
@Override
|
||||
public SolrCore getCore(String name) {
|
||||
return transientCores.get(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsCore(String name) {
|
||||
return transientCores.containsKey(name);
|
||||
}
|
||||
|
||||
// These methods allow the implementation to maintain control over the core descriptors.
|
||||
|
||||
|
||||
// This method will only be called during core discovery at startup.
|
||||
@Override
|
||||
public void addTransientDescriptor(String rawName, CoreDescriptor cd) {
|
||||
transientDescriptors.put(rawName, cd);
|
||||
}
|
||||
|
||||
// This method is used when opening cores and the like. If you want to change a core's descriptor, override this
|
||||
// method and return the current core descriptor.
|
||||
@Override
|
||||
public CoreDescriptor getTransientDescriptor(String name) {
|
||||
return transientDescriptors.get(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CoreDescriptor removeTransientDescriptor(String name) {
|
||||
return transientDescriptors.remove(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getNamesForCore(SolrCore core) {
|
||||
List<String> ret = new ArrayList<>();
|
||||
for (Map.Entry<String, SolrCore> entry : transientCores.entrySet()) {
|
||||
if (core == entry.getValue()) {
|
||||
ret.add(entry.getKey());
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Must be called in order to free resources!
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
deleteObserver(this.observer);
|
||||
}
|
||||
|
||||
|
||||
// For custom implementations to communicate arbitrary information as necessary.
|
||||
@Override
|
||||
public int getStatus(String coreName) { return 0; } //no_op for default handler.
|
||||
|
||||
@Override
|
||||
public void setStatus(String coreName, int status) {} //no_op for default handler.
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.core;
|
||||
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.Collections;
|
||||
import java.util.Locale;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.apache.solr.util.plugin.PluginInfoInitialized;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* An interface that allows custom transient caches to be maintained with different implementations
|
||||
*/
|
||||
public abstract class TransientSolrCoreCacheFactory {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
private CoreContainer coreContainer = null;
|
||||
|
||||
public abstract TransientSolrCoreCache getTransientSolrCoreCache();
|
||||
/**
|
||||
* Create a new TransientSolrCoreCacheFactory instance
|
||||
*
|
||||
* @param loader a SolrResourceLoader used to find the TransientSolrCacheFactory classes
|
||||
* @param coreContainer CoreContainer that encloses all the Solr cores.
|
||||
* @return a new, initialized TransientSolrCoreCache instance
|
||||
*/
|
||||
|
||||
public static TransientSolrCoreCacheFactory newInstance(SolrResourceLoader loader, CoreContainer coreContainer) {
|
||||
PluginInfo info = coreContainer.getConfig().getTransientCachePluginInfo();
|
||||
if (info == null) { // definition not in our solr.xml file, use default
|
||||
info = DEFAULT_TRANSIENT_SOLR_CACHE_INFO;
|
||||
}
|
||||
|
||||
try {
|
||||
// According to the docs, this returns a TransientSolrCoreCacheFactory with the default c'tor
|
||||
TransientSolrCoreCacheFactory tccf = loader.findClass(info.className, TransientSolrCoreCacheFactory.class).newInstance();
|
||||
|
||||
// OK, now we call it's init method.
|
||||
if (PluginInfoInitialized.class.isAssignableFrom(tccf.getClass()))
|
||||
PluginInfoInitialized.class.cast(tccf).init(info);
|
||||
tccf.setCoreContainer(coreContainer);
|
||||
return tccf;
|
||||
} catch (Exception e) {
|
||||
// Many things could cuse this, bad solrconfig, mis-typed class name, whatever. However, this should not
|
||||
// keep the enclosing coreContainer from instantiating, so log an error and continue.
|
||||
log.error(String.format(Locale.ROOT, "Error instantiating TransientSolrCoreCacheFactory class [%s]: %s",
|
||||
info.className, e.getMessage()));
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
public static final PluginInfo DEFAULT_TRANSIENT_SOLR_CACHE_INFO =
|
||||
new PluginInfo("transientSolrCoreCacheFactory",
|
||||
ImmutableMap.of("class", TransientSolrCoreCacheFactoryDefault.class.getName(),
|
||||
"name", TransientSolrCoreCacheFactory.class.getName()),
|
||||
null, Collections.<PluginInfo>emptyList());
|
||||
|
||||
|
||||
// Need this because the plugin framework doesn't require a PluginINfo in the init method, don't see a way to
|
||||
// pass additional parameters and we need this when we create the transient core cache, it's _really_ important.
|
||||
public void setCoreContainer(CoreContainer coreContainer) {
|
||||
this.coreContainer = coreContainer;
|
||||
}
|
||||
|
||||
public CoreContainer getCoreContainer() {
|
||||
return coreContainer;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.core;
|
||||
|
||||
public class TransientSolrCoreCacheFactoryDefault extends TransientSolrCoreCacheFactory {
|
||||
|
||||
TransientSolrCoreCache transientSolrCoreCache = null;
|
||||
|
||||
@Override
|
||||
public TransientSolrCoreCache getTransientSolrCoreCache() {
|
||||
if (transientSolrCoreCache == null) {
|
||||
transientSolrCoreCache = new TransientSolrCoreCacheDefault(getCoreContainer());
|
||||
}
|
||||
|
||||
return transientSolrCoreCache;
|
||||
}
|
||||
}
|
||||
|
|
@ -31,6 +31,11 @@
|
|||
<int name="connTimeout">${connTimeout:15000}</int>
|
||||
</shardHandlerFactory>
|
||||
|
||||
<transientCoreCacheFactory name="transientCoreCacheFactory" class="TransientSolrCoreCacheFactoryDefault">
|
||||
<int name="transientCacheSize">4</int>
|
||||
</transientCoreCacheFactory>
|
||||
|
||||
|
||||
<solrcloud>
|
||||
<str name="host">127.0.0.1</str>
|
||||
<int name="hostPort">${hostPort:8983}</int>
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ import org.apache.solr.common.util.Utils;
|
|||
import org.apache.solr.core.CloudConfig;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.core.CoreDescriptor;
|
||||
import org.apache.solr.core.SolrXmlConfig;
|
||||
import org.apache.solr.core.TransientSolrCoreCache;
|
||||
import org.apache.solr.handler.admin.CoreAdminHandler;
|
||||
import org.apache.solr.handler.component.HttpShardHandlerFactory;
|
||||
import org.apache.solr.update.UpdateShardHandler;
|
||||
|
|
@ -327,7 +329,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
|
|||
private static class MockCoreContainer extends CoreContainer {
|
||||
UpdateShardHandler updateShardHandler = new UpdateShardHandler(UpdateShardHandlerConfig.DEFAULT);
|
||||
public MockCoreContainer() {
|
||||
super((Object)null);
|
||||
super(SolrXmlConfig.fromString(null, "<solr/>"));
|
||||
this.shardHandlerFactory = new HttpShardHandlerFactory();
|
||||
this.coreAdminHandler = new CoreAdminHandler();
|
||||
}
|
||||
|
|
@ -345,6 +347,11 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
|
|||
updateShardHandler.close();
|
||||
super.shutdown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public TransientSolrCoreCache getTransientCacheHandler() {
|
||||
return transientSolrCoreCache;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -60,12 +60,12 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
|
|||
setMeUp(null);
|
||||
}
|
||||
|
||||
private Properties makeCorePropFile(String name, boolean isLazy, boolean loadOnStartup, String... extraProps) {
|
||||
private Properties makeCorePropFile(String name, boolean isTransient, boolean loadOnStartup, String... extraProps) {
|
||||
Properties props = new Properties();
|
||||
props.put(CoreDescriptor.CORE_NAME, name);
|
||||
props.put(CoreDescriptor.CORE_SCHEMA, "schema-tiny.xml");
|
||||
props.put(CoreDescriptor.CORE_CONFIG, "solrconfig-minimal.xml");
|
||||
props.put(CoreDescriptor.CORE_TRANSIENT, Boolean.toString(isLazy));
|
||||
props.put(CoreDescriptor.CORE_TRANSIENT, Boolean.toString(isTransient));
|
||||
props.put(CoreDescriptor.CORE_LOADONSTARTUP, Boolean.toString(loadOnStartup));
|
||||
props.put(CoreDescriptor.CORE_DATADIR, "${core.dataDir:stuffandnonsense}");
|
||||
|
||||
|
|
@ -140,7 +140,7 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
|
|||
try {
|
||||
|
||||
TestLazyCores.checkInCores(cc, "core1");
|
||||
TestLazyCores.checkNotInCores(cc, "lazy1", "core2", "collection1");
|
||||
TestLazyCores.checkNotInCores(cc, "lazy1", "core2");
|
||||
|
||||
// force loading of core2 and lazy1 by getting them from the CoreContainer
|
||||
try (SolrCore core1 = cc.getCore("core1");
|
||||
|
|
@ -463,4 +463,5 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
|
|||
NodeConfig absConfig = SolrXmlConfig.fromString(loader, "<solr><str name=\"coreRootDirectory\">/absolute</str></solr>");
|
||||
assertThat(absConfig.getCoreRootDirectory().toString(), not(containsString(solrHomeDirectory.getAbsolutePath())));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import org.apache.solr.handler.admin.CoreAdminHandler;
|
|||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.servlet.SolrDispatchFilter;
|
||||
import org.apache.solr.update.AddUpdateCommand;
|
||||
import org.apache.solr.update.CommitUpdateCommand;
|
||||
import org.apache.solr.update.UpdateHandler;
|
||||
|
|
@ -83,13 +84,13 @@ public class TestLazyCores extends SolrTestCaseJ4 {
|
|||
private CoreContainer init() throws Exception {
|
||||
solrHomeDirectory = createTempDir().toFile();
|
||||
|
||||
copyXmlToHome(solrHomeDirectory.getAbsoluteFile(), "solr.xml");
|
||||
for (int idx = 1; idx < 10; ++idx) {
|
||||
copyMinConf(new File(solrHomeDirectory, "collection" + idx));
|
||||
}
|
||||
|
||||
SolrResourceLoader loader = new SolrResourceLoader(solrHomeDirectory.toPath());
|
||||
NodeConfig config = new NodeConfig.NodeConfigBuilder("testNode", loader).setTransientCacheSize(4).build();
|
||||
return createCoreContainer(config, testCores);
|
||||
NodeConfig cfg = SolrDispatchFilter.loadNodeConfig(solrHomeDirectory.toPath(), null);
|
||||
return createCoreContainer(cfg, testCores);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -188,7 +189,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
|
|||
, "//result[@numFound='0']"
|
||||
);
|
||||
|
||||
checkInCores(cc, "collection4");
|
||||
checkInCores(cc, "collection1", "collection2", "collection4", "collection5");
|
||||
|
||||
core4.close();
|
||||
collection1.close();
|
||||
|
|
@ -454,11 +455,14 @@ public class TestLazyCores extends SolrTestCaseJ4 {
|
|||
// 1> produce errors as appropriate when the config or schema files are foo'd
|
||||
// 2> "self heal". That is, if the problem is corrected can the core be reloaded and used?
|
||||
// 3> that OK cores can be searched even when some cores failed to load.
|
||||
// 4> that having no solr.xml entry for transient chache handler correctly uses the default.
|
||||
@Test
|
||||
public void testBadConfigsGenerateErrors() throws Exception {
|
||||
final CoreContainer cc = initGoodAndBad(Arrays.asList("core1", "core2"),
|
||||
Arrays.asList("badSchema1", "badSchema2"),
|
||||
Arrays.asList("badConfig1", "badConfig2"));
|
||||
|
||||
|
||||
try {
|
||||
// first, did the two good cores load successfully?
|
||||
checkInCores(cc, "core1", "core2");
|
||||
|
|
@ -491,8 +495,9 @@ public class TestLazyCores extends SolrTestCaseJ4 {
|
|||
copyGoodConf("badSchema1", "schema-tiny.xml", "schema.xml");
|
||||
copyGoodConf("badSchema2", "schema-tiny.xml", "schema.xml");
|
||||
|
||||
|
||||
// This should force a reload of the cores.
|
||||
SolrCore bc1 = cc.getCore("badConfig1");
|
||||
SolrCore bc1 = cc.getCore("badConfig1");;
|
||||
SolrCore bc2 = cc.getCore("badConfig2");
|
||||
SolrCore bs1 = cc.getCore("badSchema1");
|
||||
SolrCore bs2 = cc.getCore("badSchema2");
|
||||
|
|
@ -635,16 +640,46 @@ public class TestLazyCores extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
public static void checkNotInCores(CoreContainer cc, String... nameCheck) {
|
||||
Collection<String> names = cc.getCoreNames();
|
||||
Collection<String> loadedNames = cc.getCoreNames();
|
||||
for (String name : nameCheck) {
|
||||
assertFalse("core " + name + " was found in the list of cores", names.contains(name));
|
||||
assertFalse("core " + name + " was found in the list of cores", loadedNames.contains(name));
|
||||
}
|
||||
|
||||
// There was a problem at one point exacerbated by the poor naming conventions. So parallel to loaded cores, there
|
||||
// should be the ability to get the core _names_ that are loaded as well as all the core names _possible_
|
||||
//
|
||||
// the names above should only contain loaded core names. Every name in names should be in allNames, but none of
|
||||
// the names in nameCheck should be loaded and thus should not be in names.
|
||||
|
||||
Collection<String> allNames = cc.getAllCoreNames();
|
||||
// Every core, loaded or not should be in the accumulated coredescriptors:
|
||||
List<CoreDescriptor> descriptors = cc.getCoreDescriptors();
|
||||
|
||||
assertEquals("There should be as many coreDescriptors as coreNames", allNames.size(), descriptors.size());
|
||||
for (CoreDescriptor desc : descriptors) {
|
||||
assertTrue("Name should have a corresponding descriptor", allNames.contains(desc.getName()));
|
||||
}
|
||||
|
||||
// First check that all loaded cores are in allNames.
|
||||
for (String name : loadedNames) {
|
||||
assertTrue("Loaded core " + name + " should have been found in the list of all possible core names",
|
||||
allNames.contains(name));
|
||||
}
|
||||
|
||||
for (String name : nameCheck) {
|
||||
assertTrue("Not-currently-loaded core " + name + " should have been found in the list of all possible core names",
|
||||
allNames.contains(name));
|
||||
}
|
||||
}
|
||||
|
||||
public static void checkInCores(CoreContainer cc, String... nameCheck) {
|
||||
Collection<String> names = cc.getCoreNames();
|
||||
Collection<String> loadedNames = cc.getCoreNames();
|
||||
|
||||
assertEquals("There whould be exactly as many loaded cores as loaded names returned. ",
|
||||
loadedNames.size(), nameCheck.length);
|
||||
|
||||
for (String name : nameCheck) {
|
||||
assertTrue("core " + name + " was not found in the list of cores", names.contains(name));
|
||||
assertTrue("core " + name + " was not found in the list of cores", loadedNames.contains(name));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2002,8 +2002,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
|||
FileUtils.copyFile(new File(top, "solrconfig.snippet.randomindexconfig.xml"), new File(subHome, "solrconfig.snippet.randomindexconfig.xml"));
|
||||
}
|
||||
|
||||
// Creates minimal full setup, including the old solr.xml file that used to be hard coded in ConfigSolrXmlOld
|
||||
// TODO: remove for 5.0
|
||||
// Creates minimal full setup, including solr.xml
|
||||
public static void copyMinFullSetup(File dstRoot) throws IOException {
|
||||
if (! dstRoot.exists()) {
|
||||
assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
|
||||
|
|
@ -2013,6 +2012,15 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
|||
copyMinConf(dstRoot);
|
||||
}
|
||||
|
||||
// Just copies the file indicated to the tmp home directory naming it "solr.xml"
|
||||
public static void copyXmlToHome(File dstRoot, String fromFile) throws IOException {
|
||||
if (! dstRoot.exists()) {
|
||||
assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
|
||||
}
|
||||
File xmlF = new File(SolrTestCaseJ4.TEST_HOME(), fromFile);
|
||||
FileUtils.copyFile(xmlF, new File(dstRoot, "solr.xml"));
|
||||
|
||||
}
|
||||
// Creates a consistent configuration, _including_ solr.xml at dstRoot. Creates collection1/conf and copies
|
||||
// the stock files in there.
|
||||
|
||||
|
|
@ -2020,7 +2028,6 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
|||
if (!dstRoot.exists()) {
|
||||
assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
|
||||
}
|
||||
|
||||
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(dstRoot, "solr.xml"));
|
||||
|
||||
File subHome = new File(dstRoot, collection + File.separator + "conf");
|
||||
|
|
|
|||
Loading…
Reference in New Issue