Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4749: Clean up and refactor CoreContainer code around solr.xml and SolrCore management. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1470674 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-04-22 19:35:10 +00:00
parent c5a5a77336
commit 87e248f2d5
11 changed files with 1514 additions and 1287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -78,6 +78,9 @@ Other Changes
* SOLR-4738: Update Jetty to 8.1.10.v20130312 (Mark Miller, Robert Muir)
* SOLR-4749: Clean up and refactor CoreContainer code around solr.xml and SolrCore
management. (Mark Miller)
================== 4.3.0 ==================
Versions of Major Components

13
solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
View File

@ -20,10 +20,13 @@ import org.apache.commons.cli.PosixParser;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.cloud.OnReconnect;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.core.Config;
import org.apache.solr.core.ConfigSolr;
import org.apache.solr.core.ConfigSolrXml;
import org.apache.solr.core.ConfigSolrXmlOld;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.zookeeper.KeeperException;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
/*
@ -181,7 +184,15 @@ public class ZkCLI {
ConfigSolr cfg;
try {
cfg = new ConfigSolrXml(loader, null, is, null, false, null);
Config config = new Config(loader, null, new InputSource(is), null, false);
boolean oldStyle = (config.getNode("solr/cores", false) != null);
// cfg = new ConfigSolrXml(loader, null, is, null, false, this);
if (oldStyle) {
cfg = new ConfigSolrXmlOld(config, null);
} else {
cfg = new ConfigSolrXml(config, null);
}
} finally {
IOUtils.closeQuietly(is);
}

27
solr/core/src/java/org/apache/solr/core/Config.java
View File

@ -37,11 +37,16 @@ import javax.xml.namespace.QName;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.dom.DOMSource;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
@ -75,18 +80,6 @@ public class Config {
this( loader, name, null, null );
}
/**
* For the transition from using solr.xml to solr.properties, see SOLR-4196. Remove
* for 5.0, thus it's already deprecated
* @param loader - Solr resource loader
* @param cfg - SolrConfig, for backwards compatability with solr.xml layer.
* @throws TransformerException if the XML file is mal-formed
*/
@Deprecated
public Config(SolrResourceLoader loader, Config cfg) throws TransformerException {
this(loader, null, ConfigSolrXml.copyDoc(cfg.getDocument()));
}
public Config(SolrResourceLoader loader, String name, InputSource is, String prefix) throws ParserConfigurationException, IOException, SAXException
{
this(loader, name, is, prefix, true);
@ -164,6 +157,16 @@ public class Config {
this.loader = loader;
}
private static Document copyDoc(Document doc) throws TransformerException {
TransformerFactory tfactory = TransformerFactory.newInstance();
Transformer tx = tfactory.newTransformer();
DOMSource source = new DOMSource(doc);
DOMResult result = new DOMResult();
tx.transform(source, result);
return (Document) result.getNode();
}
/**
* @since solr 1.3
*/

182
solr/core/src/java/org/apache/solr/core/ConfigSolr.java
View File

@ -17,26 +17,30 @@ package org.apache.solr.core;
* limitations under the License.
*/
import org.apache.solr.cloud.ZkController;
import org.apache.solr.handler.component.ShardHandlerFactory;
import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* ConfigSolr is a new interface to aid us in obsoleting solr.xml and replacing it with solr.properties. The problem here
* is that the Config class is used for _all_ the xml file, e.g. solrconfig.xml and we can't mess with _that_ as part
* of this issue. Primarily used in CoreContainer at present.
* <p/>
* This is already deprecated, it's only intended to exist for while transitioning to properties-based replacement for
* solr.xml
*
* @since solr 4.3
*/
public interface ConfigSolr {
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.PropertiesUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public abstract class ConfigSolr {
protected static Logger log = LoggerFactory.getLogger(ConfigSolr.class);
public final static String CORE_PROP_FILE = "core.properties";
public final static String SOLR_XML_FILE = "solr.xml";
// Ugly for now, but we'll at least be able to centralize all of the differences between 4x and 5x.
public static enum CfgProp {
SOLR_ADMINHANDLER,
@ -67,48 +71,118 @@ public interface ConfigSolr {
SOLR_PERSISTENT,
SOLR_CORES_DEFAULT_CORE_NAME,
SOLR_ADMINPATH
};
}
public final static String CORE_PROP_FILE = "core.properties";
public final static String SOLR_XML_FILE = "solr.xml";
public int getInt(CfgProp prop, int def);
public boolean getBool(CfgProp prop,boolean defValue);
public String get(CfgProp prop, String def);
public String getOrigProp(CfgProp prop, String def);
public void substituteProperties();
public ShardHandlerFactory initShardHandler();
public Properties getSolrProperties(String context);
public SolrConfig getSolrConfigFromZk(ZkController zkController, String zkConfigName, String solrConfigFileName,
SolrResourceLoader resourceLoader);
public void initPersist();
public void addPersistCore(String coreName, Properties attribs, Map<String, String> props);
public void addPersistAllCores(Properties containerProperties, Map<String, String> rootSolrAttribs, Map<String, String> coresAttribs,
File file);
public String getCoreNameFromOrig(String origCoreName, SolrResourceLoader loader, String coreName);
public List<String> getAllCoreNames();
public String getProperty(String coreName, String property, String defaultVal);
public Properties readCoreProperties(String coreName);
public Map<String, String> readCoreAttributes(String coreName);
protected Config config;
protected Map<CfgProp, String> propMap = new HashMap<CfgProp, String>();
protected final Map<String, String> badConfigCores = new HashMap<String, String>();
public ConfigSolr(Config config) {
this.config = config;
}
public Config getConfig() {
return config;
}
// If the core is not to be loaded (say two cores defined with the same name or with the same data dir), return
// the reason. If it's OK to load the core, return null.
public String getBadConfigCoreMessage(String name);
public String getBadConfigCoreMessage(String name) {
return badConfigCores.get(name);
}
public int getInt(CfgProp prop, int def) {
String val = propMap.get(prop);
if (val != null) val = PropertiesUtil.substituteProperty(val, null);
return (val == null) ? def : Integer.parseInt(val);
}
public boolean getBool(CfgProp prop, boolean defValue) {
String val = propMap.get(prop);
if (val != null) val = PropertiesUtil.substituteProperty(val, null);
return (val == null) ? defValue : Boolean.parseBoolean(val);
}
public String get(CfgProp prop, String def) {
String val = propMap.get(prop);
if (val != null) val = PropertiesUtil.substituteProperty(val, null);
return (val == null) ? def : val;
}
// For saving the original property, ${} syntax and all.
public String getOrigProp(CfgProp prop, String def) {
String val = propMap.get(prop);
return (val == null) ? def : val;
}
public Properties getSolrProperties(String path) {
try {
return readProperties(((NodeList) config.evaluate(
path, XPathConstants.NODESET)).item(0));
} catch (Throwable e) {
SolrException.log(log, null, e);
}
return null;
}
protected Properties readProperties(Node node) throws XPathExpressionException {
XPath xpath = config.getXPath();
NodeList props = (NodeList) xpath.evaluate("property", node, XPathConstants.NODESET);
Properties properties = new Properties();
for (int i = 0; i < props.getLength(); i++) {
Node prop = props.item(i);
properties.setProperty(DOMUtil.getAttr(prop, "name"), DOMUtil.getAttr(prop, "value"));
}
return properties;
}
public abstract void substituteProperties();
public abstract String getCoreNameFromOrig(String origCoreName, SolrResourceLoader loader, String coreName);
public abstract List<String> getAllCoreNames();
public abstract String getProperty(String coreName, String property, String defaultVal);
public abstract Properties readCoreProperties(String coreName);
public abstract Map<String, String> readCoreAttributes(String coreName);
public boolean is50OrLater();
}
// It's mightily convenient to have all of the original path names and property
// values when persisting cores, so
// this little convenience class is just for that.
// Also, let's keep track of anything we added here, especially the instance dir
// for persistence purposes. We don't
// want, for instance, to persist instanceDir if it was not specified
// originally.
//
// I suspect that for persistence purposes, we may want to expand this idea to
// record, say, ${blah}
class CoreDescriptorPlus {
private CoreDescriptor coreDescriptor;
private String filePath;
private Properties propsOrig; // TODO: 5.0. Remove this since it's only really
// used for persisting.
CoreDescriptorPlus(String filePath, CoreDescriptor descriptor,
Properties propsOrig) {
coreDescriptor = descriptor;
this.filePath = filePath;
this.propsOrig = propsOrig;
}
CoreDescriptor getCoreDescriptor() {
return coreDescriptor;
}
String getFilePath() {
return filePath;
}
Properties getPropsOrig() {
return propsOrig;
}
}

685
solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
View File

@ -17,153 +17,86 @@ package org.apache.solr.core;
* limitations under the License.
*/
import org.apache.commons.io.IOUtils;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.PropertiesUtil;
import org.apache.solr.util.SystemIdResolver;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.dom.DOMSource;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.io.IOUtils;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.PropertiesUtil;
import org.apache.solr.util.SystemIdResolver;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
/**
* ConfigSolrXml
* <p/>
* This class is entirely to localize the dealing with specific issues when transitioning from old-style solr.xml
* to new-style xml, enumeration/discovery of defined cores. See SOLR-4196 for background.
* <p/>
* @since solr 4.3
*
* It's a bit twisted, but we decided to NOT do the solr.properties switch. But since there's already an interface
* it makes sense to leave it in so we can use other methods of providing the Solr information that is contained
* in solr.xml. Perhaps something form SolrCloud in the future?
*
*/
public class ConfigSolrXml extends Config implements ConfigSolr {
private boolean is50OrLater = false;
public class ConfigSolrXml extends ConfigSolr {
protected static Logger log = LoggerFactory.getLogger(ConfigSolrXml.class);
private final Map<String, CoreDescriptorPlus> coreDescriptorPlusMap = new HashMap<String, CoreDescriptorPlus>();
private NodeList coreNodes = null;
private final Map<String, String> badConfigCores = new HashMap<String, String>();
// List of cores that we should _never_ load. Ones with dup names or duplicate datadirs or...
private Map<CfgProp, String> propMap = new HashMap<CfgProp, String>();
public ConfigSolrXml(SolrResourceLoader loader, String name, InputStream is, String prefix,
boolean subProps, CoreContainer container)
public ConfigSolrXml(Config config, CoreContainer container)
throws ParserConfigurationException, IOException, SAXException {
super(loader, name, new InputSource(is), prefix, subProps);
init(container);
}
public ConfigSolrXml(SolrResourceLoader loader, Config cfg, CoreContainer container)
throws TransformerException, IOException {
super(loader, null, copyDoc(cfg.getDocument())); // Mimics a call from CoreContainer.
super(config);
init(container);
}
private void init(CoreContainer container) throws IOException {
is50OrLater = getNode("solr/cores", false) == null;
// Do sanity checks, old and new style. Pretty exhaustive for now, but want to hammer this.
// TODO: 5.0 maybe remove this checking, it's mostly for correctness as we make this transition.
if (is50OrLater()) {
failIfFound("solr/@coreLoadThreads");
failIfFound("solr/@persist");
failIfFound("solr/@sharedLib");
failIfFound("solr/@zkHost");
failIfFound("solr/logging/@class");
failIfFound("solr/logging/@enabled");
failIfFound("solr/logging/watcher/@size");
failIfFound("solr/logging/watcher/@threshold");
failIfFound("solr/cores/@adminHandler");
failIfFound("solr/cores/@distribUpdateConnTimeout");
failIfFound("solr/cores/@distribUpdateSoTimeout");
failIfFound("solr/cores/@host");
failIfFound("solr/cores/@hostContext");
failIfFound("solr/cores/@hostPort");
failIfFound("solr/cores/@leaderVoteWait");
failIfFound("solr/cores/@managementPath");
failIfFound("solr/cores/@shareSchema");
failIfFound("solr/cores/@transientCacheSize");
failIfFound("solr/cores/@zkClientTimeout");
// These have no counterpart in 5.0, asking for any of these in Solr 5.0 will result in an error being
// thrown.
failIfFound("solr/cores/@defaultCoreName");
failIfFound("solr/@persistent");
failIfFound("solr/cores/@adminPath");
} else {
failIfFound("solr/str[@name='adminHandler']");
failIfFound("solr/int[@name='coreLoadThreads']");
failIfFound("solr/str[@name='coreRootDirectory']");
failIfFound("solr/solrcloud/int[@name='distribUpdateConnTimeout']");
failIfFound("solr/solrcloud/int[@name='distribUpdateSoTimeout']");
failIfFound("solr/solrcloud/str[@name='host']");
failIfFound("solr/solrcloud/str[@name='hostContext']");
failIfFound("solr/solrcloud/int[@name='hostPort']");
failIfFound("solr/solrcloud/int[@name='leaderVoteWait']");
failIfFound("solr/str[@name='managementPath']");
failIfFound("solr/str[@name='sharedLib']");
failIfFound("solr/str[@name='shareSchema']");
failIfFound("solr/int[@name='transientCacheSize']");
failIfFound("solr/solrcloud/int[@name='zkClientTimeout']");
failIfFound("solr/solrcloud/int[@name='zkHost']");
failIfFound("solr/logging/str[@name='class']");
failIfFound("solr/logging/str[@name='enabled']");
failIfFound("solr/logging/watcher/int[@name='size']");
failIfFound("solr/logging/watcher/int[@name='threshold']");
}
// Do sanity checks - we don't want to find old style config
failIfFound("solr/@coreLoadThreads");
failIfFound("solr/@persist");
failIfFound("solr/@sharedLib");
failIfFound("solr/@zkHost");
failIfFound("solr/logging/@class");
failIfFound("solr/logging/@enabled");
failIfFound("solr/logging/watcher/@size");
failIfFound("solr/logging/watcher/@threshold");
failIfFound("solr/cores/@adminHandler");
failIfFound("solr/cores/@distribUpdateConnTimeout");
failIfFound("solr/cores/@distribUpdateSoTimeout");
failIfFound("solr/cores/@host");
failIfFound("solr/cores/@hostContext");
failIfFound("solr/cores/@hostPort");
failIfFound("solr/cores/@leaderVoteWait");
failIfFound("solr/cores/@managementPath");
failIfFound("solr/cores/@shareSchema");
failIfFound("solr/cores/@transientCacheSize");
failIfFound("solr/cores/@zkClientTimeout");
// These have no counterpart in 5.0, asking for any of these in Solr 5.0
// will result in an error being
// thrown.
failIfFound("solr/cores/@defaultCoreName");
failIfFound("solr/@persistent");
failIfFound("solr/cores/@adminPath");
fillPropMap();
initCoreList(container);
}
private void failIfFound(String xPath) {
if (getVal(xPath, false) != null) {
if (config.getVal(xPath, false) != null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Should not have found " + xPath +
" solr.xml may be a mix of old and new style formats.");
}
@ -171,7 +104,7 @@ public class ConfigSolrXml extends Config implements ConfigSolr {
// We can do this in 5.0 when we read the solr.xml since we don't need to keep the original around for persistence.
private String doSub(String path) {
String val = getVal(path, false);
String val = config.getVal(path, false);
if (val != null) {
val = PropertiesUtil.substituteProperty(val, null);
}
@ -179,228 +112,49 @@ public class ConfigSolrXml extends Config implements ConfigSolr {
}
private void fillPropMap() {
if (is50OrLater) { // Can do the prop subs early here since we don't need to preserve them for persistence.
propMap.put(CfgProp.SOLR_ADMINHANDLER, doSub("solr/str[@name='adminHandler']"));
propMap.put(CfgProp.SOLR_CORELOADTHREADS, doSub("solr/int[@name='coreLoadThreads']"));
propMap.put(CfgProp.SOLR_COREROOTDIRECTORY, doSub("solr/str[@name='coreRootDirectory']"));
propMap.put(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, doSub("solr/solrcloud/int[@name='distribUpdateConnTimeout']"));
propMap.put(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, doSub("solr/solrcloud/int[@name='distribUpdateSoTimeout']"));
propMap.put(CfgProp.SOLR_HOST, doSub("solr/solrcloud/str[@name='host']"));
propMap.put(CfgProp.SOLR_HOSTCONTEXT, doSub("solr/solrcloud/str[@name='hostContext']"));
propMap.put(CfgProp.SOLR_HOSTPORT, doSub("solr/solrcloud/int[@name='hostPort']"));
propMap.put(CfgProp.SOLR_LEADERVOTEWAIT, doSub("solr/solrcloud/int[@name='leaderVoteWait']"));
propMap.put(CfgProp.SOLR_MANAGEMENTPATH, doSub("solr/str[@name='managementPath']"));
propMap.put(CfgProp.SOLR_SHAREDLIB, doSub("solr/str[@name='sharedLib']"));
propMap.put(CfgProp.SOLR_SHARESCHEMA, doSub("solr/str[@name='shareSchema']"));
propMap.put(CfgProp.SOLR_TRANSIENTCACHESIZE, doSub("solr/int[@name='transientCacheSize']"));
propMap.put(CfgProp.SOLR_ZKCLIENTTIMEOUT, doSub("solr/solrcloud/int[@name='zkClientTimeout']"));
propMap.put(CfgProp.SOLR_ZKHOST, doSub("solr/solrcloud/str[@name='zkHost']"));
propMap.put(CfgProp.SOLR_ADMINHANDLER, doSub("solr/str[@name='adminHandler']"));
propMap.put(CfgProp.SOLR_CORELOADTHREADS, doSub("solr/int[@name='coreLoadThreads']"));
propMap.put(CfgProp.SOLR_COREROOTDIRECTORY, doSub("solr/str[@name='coreRootDirectory']"));
propMap.put(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, doSub("solr/solrcloud/int[@name='distribUpdateConnTimeout']"));
propMap.put(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, doSub("solr/solrcloud/int[@name='distribUpdateSoTimeout']"));
propMap.put(CfgProp.SOLR_HOST, doSub("solr/solrcloud/str[@name='host']"));
propMap.put(CfgProp.SOLR_HOSTCONTEXT, doSub("solr/solrcloud/str[@name='hostContext']"));
propMap.put(CfgProp.SOLR_HOSTPORT, doSub("solr/solrcloud/int[@name='hostPort']"));
propMap.put(CfgProp.SOLR_LEADERVOTEWAIT, doSub("solr/solrcloud/int[@name='leaderVoteWait']"));
propMap.put(CfgProp.SOLR_MANAGEMENTPATH, doSub("solr/str[@name='managementPath']"));
propMap.put(CfgProp.SOLR_SHAREDLIB, doSub("solr/str[@name='sharedLib']"));
propMap.put(CfgProp.SOLR_SHARESCHEMA, doSub("solr/str[@name='shareSchema']"));
propMap.put(CfgProp.SOLR_TRANSIENTCACHESIZE, doSub("solr/int[@name='transientCacheSize']"));
propMap.put(CfgProp.SOLR_ZKCLIENTTIMEOUT, doSub("solr/solrcloud/int[@name='zkClientTimeout']"));
propMap.put(CfgProp.SOLR_ZKHOST, doSub("solr/solrcloud/str[@name='zkHost']"));
propMap.put(CfgProp.SOLR_LOGGING_CLASS, doSub("solr/logging/str[@name='class']"));
propMap.put(CfgProp.SOLR_LOGGING_ENABLED, doSub("solr/logging/str[@name='enabled']"));
propMap.put(CfgProp.SOLR_LOGGING_WATCHER_SIZE, doSub("solr/logging/watcher/int[@name='size']"));
propMap.put(CfgProp.SOLR_LOGGING_WATCHER_THRESHOLD, doSub("solr/logging/watcher/int[@name='threshold']"));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_CLASS, doSub("solr/shardHandlerFactory/@class"));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_NAME, doSub("solr/shardHandlerFactory/@name"));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_CONNTIMEOUT, doSub("solr/shardHandlerFactory/int[@name='connTimeout']"));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_SOCKETTIMEOUT, doSub("solr/shardHandlerFactory/int[@name='socketTimeout']"));
} else {
propMap.put(CfgProp.SOLR_CORELOADTHREADS, getVal("solr/@coreLoadThreads", false));
propMap.put(CfgProp.SOLR_SHAREDLIB, getVal("solr/@sharedLib", false));
propMap.put(CfgProp.SOLR_ZKHOST, getVal("solr/@zkHost", false));
propMap.put(CfgProp.SOLR_LOGGING_CLASS, getVal("solr/logging/@class", false));
propMap.put(CfgProp.SOLR_LOGGING_ENABLED, getVal("solr/logging/@enabled", false));
propMap.put(CfgProp.SOLR_LOGGING_WATCHER_SIZE, getVal("solr/logging/watcher/@size", false));
propMap.put(CfgProp.SOLR_LOGGING_WATCHER_THRESHOLD, getVal("solr/logging/watcher/@threshold", false));
propMap.put(CfgProp.SOLR_ADMINHANDLER, getVal("solr/cores/@adminHandler", false));
propMap.put(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, getVal("solr/cores/@distribUpdateConnTimeout", false));
propMap.put(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, getVal("solr/cores/@distribUpdateSoTimeout", false));
propMap.put(CfgProp.SOLR_HOST, getVal("solr/cores/@host", false));
propMap.put(CfgProp.SOLR_HOSTCONTEXT, getVal("solr/cores/@hostContext", false));
propMap.put(CfgProp.SOLR_HOSTPORT, getVal("solr/cores/@hostPort", false));
propMap.put(CfgProp.SOLR_LEADERVOTEWAIT, getVal("solr/cores/@leaderVoteWait", false));
propMap.put(CfgProp.SOLR_MANAGEMENTPATH, getVal("solr/cores/@managementPath", false));
propMap.put(CfgProp.SOLR_SHARESCHEMA, getVal("solr/cores/@shareSchema", false));
propMap.put(CfgProp.SOLR_TRANSIENTCACHESIZE, getVal("solr/cores/@transientCacheSize", false));
propMap.put(CfgProp.SOLR_ZKCLIENTTIMEOUT, getVal("solr/cores/@zkClientTimeout", false));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_CLASS, getVal("solr/shardHandlerFactory/@class", false));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_NAME, getVal("solr/shardHandlerFactory/@name", false));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_CONNTIMEOUT, getVal( "solr/shardHandlerFactory/int[@connTimeout]", false));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_SOCKETTIMEOUT, getVal("solr/shardHandlerFactory/int[@socketTimeout]", false));
// These have no counterpart in 5.0, asking, for any of these in Solr 5.0 will result in an error being
// thrown.
propMap.put(CfgProp.SOLR_CORES_DEFAULT_CORE_NAME, getVal("solr/cores/@defaultCoreName", false));
propMap.put(CfgProp.SOLR_PERSISTENT, getVal("solr/@persistent", false));
propMap.put(CfgProp.SOLR_ADMINPATH, getVal("solr/cores/@adminPath", false));
}
propMap.put(CfgProp.SOLR_LOGGING_CLASS, doSub("solr/logging/str[@name='class']"));
propMap.put(CfgProp.SOLR_LOGGING_ENABLED, doSub("solr/logging/str[@name='enabled']"));
propMap.put(CfgProp.SOLR_LOGGING_WATCHER_SIZE, doSub("solr/logging/watcher/int[@name='size']"));
propMap.put(CfgProp.SOLR_LOGGING_WATCHER_THRESHOLD, doSub("solr/logging/watcher/int[@name='threshold']"));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_CLASS, doSub("solr/shardHandlerFactory/@class"));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_NAME, doSub("solr/shardHandlerFactory/@name"));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_CONNTIMEOUT, doSub("solr/shardHandlerFactory/int[@name='connTimeout']"));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_SOCKETTIMEOUT, doSub("solr/shardHandlerFactory/int[@name='socketTimeout']"));
}
private void initCoreList(CoreContainer container) throws IOException {
if (is50OrLater) {
if (container != null) { //TODO: 5.0. Yet another bit of nonsense only because of the test harness.
synchronized (coreDescriptorPlusMap) {
String coreRoot = get(CfgProp.SOLR_COREROOTDIRECTORY, container.getSolrHome());
walkFromHere(new File(coreRoot), container, new HashMap<String, String>(), new HashMap<String, String>());
}
}
} else {
coreNodes = (NodeList) evaluate("solr/cores/core",
XPathConstants.NODESET);
// Check a couple of error conditions
Set<String> names = new HashSet<String>(); // for duplicate names
Map<String, String> dirs = new HashMap<String, String>(); // for duplicate data dirs.
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
String name = DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null);
String dataDir = DOMUtil.getAttr(node, CoreDescriptor.CORE_DATADIR, null);
if (name != null) {
if (! names.contains(name)) {
names.add(name);
} else {
String msg = String.format(Locale.ROOT, "More than one core defined for core named %s", name);
log.error(msg);
}
}
if (dataDir != null) {
if (! dirs.containsKey(dataDir)) {
dirs.put(dataDir, name);
} else {
String msg = String.format(Locale.ROOT, "More than one core points to data dir %s. They are in %s and %s",
dataDir, dirs.get(dataDir), name);
log.warn(msg);
}
}
if (container != null) { // TODO: 5.0. Yet another bit of nonsense only
// because of the test harness.
synchronized (coreDescriptorPlusMap) {
String coreRoot = get(CfgProp.SOLR_COREROOTDIRECTORY,
container.getSolrHome());
walkFromHere(new File(coreRoot), container,
new HashMap<String,String>(), new HashMap<String,String>());
}
}
}
@Override
public String getBadConfigCoreMessage(String name) {
return badConfigCores.get(name);
}
public static Document copyDoc(Document doc) throws TransformerException {
TransformerFactory tfactory = TransformerFactory.newInstance();
Transformer tx = tfactory.newTransformer();
DOMSource source = new DOMSource(doc);
DOMResult result = new DOMResult();
tx.transform(source, result);
return (Document) result.getNode();
}
//TODO: For 5.0, you shouldn't have to do the sbustituteProperty, this is anothe bit
// of awkward back-compat due to persistence.
@Override
public int getInt(CfgProp prop, int def) {
String val = propMap.get(prop);
if (val != null) val = PropertiesUtil.substituteProperty(val, null);
return (val == null) ? def : Integer.parseInt(val);
}
@Override
public boolean getBool(CfgProp prop, boolean defValue) {
String val = propMap.get(prop);
if (val != null) val = PropertiesUtil.substituteProperty(val, null);
return (val == null) ? defValue : Boolean.parseBoolean(val);
}
@Override
public String get(CfgProp prop, String def) {
String val = propMap.get(prop);
if (val != null) val = PropertiesUtil.substituteProperty(val, null);
return (val == null) ? def : val;
}
// For saving the original property, ${} syntax and all.
@Override
public String getOrigProp(CfgProp prop, String def) {
String val = propMap.get(prop);
return (val == null) ? def : val;
}
public ShardHandlerFactory initShardHandler() {
PluginInfo info = null;
Node shfn = getNode("solr/cores/shardHandlerFactory", false);
if (shfn != null) {
info = new PluginInfo(shfn, "shardHandlerFactory", false, true);
} else {
Map m = new HashMap();
m.put("class", HttpShardHandlerFactory.class.getName());
info = new PluginInfo("shardHandlerFactory", m, null, Collections.<PluginInfo>emptyList());
}
ShardHandlerFactory fac;
try {
fac = getResourceLoader().findClass(info.className, ShardHandlerFactory.class).newInstance();
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error instantiating shardHandlerFactory class " + info.className);
}
if (fac instanceof PluginInfoInitialized) {
((PluginInfoInitialized) fac).init(info);
}
return fac;
}
@Override
public Properties getSolrProperties(String path) {
try {
return readProperties(((NodeList) evaluate(
path, XPathConstants.NODESET)).item(0));
} catch (Throwable e) {
SolrException.log(log, null, e);
}
return null;
}
Properties readProperties(Node node) throws XPathExpressionException {
XPath xpath = getXPath();
NodeList props = (NodeList) xpath.evaluate("property", node, XPathConstants.NODESET);
Properties properties = new Properties();
for (int i = 0; i < props.getLength(); i++) {
Node prop = props.item(i);
properties.setProperty(DOMUtil.getAttr(prop, "name"), DOMUtil.getAttr(prop, "value"));
}
return properties;
}
@Override
public Map<String, String> readCoreAttributes(String coreName) {
Map<String, String> attrs = new HashMap<String, String>();
if (is50OrLater) {
return attrs; // this is a no-op.... intentionally
}
synchronized (coreNodes) {
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
if (coreName.equals(DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null))) {
NamedNodeMap attributes = node.getAttributes();
for (int i = 0; i < attributes.getLength(); i++) {
Node attribute = attributes.item(i);
String val = attribute.getNodeValue();
if (CoreDescriptor.CORE_DATADIR.equals(attribute.getNodeName()) ||
CoreDescriptor.CORE_INSTDIR.equals(attribute.getNodeName())) {
if (val.indexOf('$') == -1) {
val = (val != null && !val.endsWith("/")) ? val + '/' : val;
}
}
attrs.put(attribute.getNodeName(), val);
}
return attrs;
}
}
}
return attrs;
public Map<String,String> readCoreAttributes(String coreName) {
Map<String,String> attrs = new HashMap<String,String>();
return attrs; // this is a no-op.... intentionally
}
// Basic recursive tree walking, looking for "core.properties" files. Once one is found, we'll stop going any
@ -482,117 +236,45 @@ public class ConfigSolrXml extends Config implements ConfigSolr {
coreDescriptorPlusMap.remove(desc.getName());
}
public IndexSchema getSchemaFromZk(ZkController zkController, String zkConfigName, String schemaName,
SolrConfig config)
throws KeeperException, InterruptedException {
byte[] configBytes = zkController.getConfigFileData(zkConfigName, schemaName);
InputSource is = new InputSource(new ByteArrayInputStream(configBytes));
is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(schemaName));
IndexSchema schema = new IndexSchema(config, schemaName, is);
return schema;
}
@Override
public SolrConfig getSolrConfigFromZk(ZkController zkController, String zkConfigName, String solrConfigFileName,
SolrResourceLoader resourceLoader) {
SolrConfig cfg = null;
try {
byte[] config = zkController.getConfigFileData(zkConfigName, solrConfigFileName);
InputSource is = new InputSource(new ByteArrayInputStream(config));
is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(solrConfigFileName));
cfg = solrConfigFileName == null ? new SolrConfig(
resourceLoader, SolrConfig.DEFAULT_CONF_FILE, is) : new SolrConfig(
resourceLoader, solrConfigFileName, is);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"getSolrConfigFromZK failed for " + zkConfigName + " " + solrConfigFileName, e);
public String getCoreNameFromOrig(String origCoreName,
SolrResourceLoader loader, String coreName) {
// first look for an exact match
for (Map.Entry<String,CoreDescriptorPlus> ent : coreDescriptorPlusMap
.entrySet()) {
String name = ent.getValue().getCoreDescriptor()
.getProperty(CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(name)) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
return cfg;
}
static List<SolrXMLSerializer.SolrCoreXMLDef> solrCoreXMLDefs = new ArrayList<SolrXMLSerializer.SolrCoreXMLDef>();
// Do this when re-using a ConfigSolrXml.
// These two methods are part of SOLR-4196 and are awkward, should go away with 5.0
@Override
public void initPersist() {
initPersistStatic();
}
public static void initPersistStatic() {
solrCoreXMLDefs = new ArrayList<SolrXMLSerializer.SolrCoreXMLDef>();
solrXMLSerializer = new SolrXMLSerializer();
}
@Override
public void addPersistCore(String coreName, Properties attribs, Map<String, String> props) {
addPersistCore(attribs, props);
}
static void addPersistCore(Properties props, Map<String, String> attribs) {
SolrXMLSerializer.SolrCoreXMLDef solrCoreXMLDef = new SolrXMLSerializer.SolrCoreXMLDef();
solrCoreXMLDef.coreAttribs = attribs;
solrCoreXMLDef.coreProperties = props;
solrCoreXMLDefs.add(solrCoreXMLDef);
}
private static SolrXMLSerializer solrXMLSerializer = new SolrXMLSerializer();
@Override
public void addPersistAllCores(Properties containerProperties, Map<String, String> rootSolrAttribs, Map<String, String> coresAttribs,
File file) {
addPersistAllCoresStatic(containerProperties, rootSolrAttribs, coresAttribs, file);
}
// Fortunately, we don't iterate over these too often, so the waste is probably tolerable.
@Override
public String getCoreNameFromOrig(String origCoreName, SolrResourceLoader loader, String coreName) {
if (is50OrLater) {
// first look for an exact match
for (Map.Entry<String, CoreDescriptorPlus> ent : coreDescriptorPlusMap.entrySet()) {
String name = ent.getValue().getCoreDescriptor().getProperty(CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(name)) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
for (Map.Entry<String, CoreDescriptorPlus> ent : coreDescriptorPlusMap.entrySet()) {
String name = ent.getValue().getCoreDescriptor().getProperty(CoreDescriptor.CORE_NAME, null);
// see if we match with substitution
if (origCoreName.equals(PropertiesUtil.substituteProperty(name, loader.getCoreProperties()))) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
} else {
// look for an existing node
synchronized (coreNodes) {
// first look for an exact match
Node coreNode = null;
for (int i = 0; i < coreNodes.getLength(); i++) {
Node node = coreNodes.item(i);
String name = DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(name)) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
if (coreNode == null) {
// see if we match with substitution
for (int i = 0; i < coreNodes.getLength(); i++) {
Node node = coreNodes.item(i);
String name = DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(PropertiesUtil.substituteProperty(name,
loader.getCoreProperties()))) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
for (Map.Entry<String,CoreDescriptorPlus> ent : coreDescriptorPlusMap
.entrySet()) {
String name = ent.getValue().getCoreDescriptor()
.getProperty(CoreDescriptor.CORE_NAME, null);
// see if we match with substitution
if (origCoreName.equals(PropertiesUtil.substituteProperty(name,
loader.getCoreProperties()))) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
return null;
@ -600,66 +282,30 @@ public class ConfigSolrXml extends Config implements ConfigSolr {
@Override
public List<String> getAllCoreNames() {
List<String> ret = new ArrayList<String>();
if (is50OrLater) {
ret = new ArrayList<String>(coreDescriptorPlusMap.keySet());
} else {
synchronized (coreNodes) {
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
ret.add(DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null));
}
}
}
List<String> ret = new ArrayList<String>(coreDescriptorPlusMap.keySet());
return ret;
}
@Override
public String getProperty(String coreName, String property, String defaultVal) {
if (is50OrLater) {
CoreDescriptorPlus plus = coreDescriptorPlusMap.get(coreName);
if (plus == null) return defaultVal;
CoreDescriptor desc = plus.getCoreDescriptor();
if (desc == null) return defaultVal;
return desc.getProperty(property, defaultVal);
} else {
synchronized (coreNodes) {
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
if (coreName.equals(DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null))) {
return DOMUtil.getAttr(node, property, defaultVal);
}
}
}
return defaultVal;
}
CoreDescriptorPlus plus = coreDescriptorPlusMap.get(coreName);
if (plus == null) return defaultVal;
CoreDescriptor desc = plus.getCoreDescriptor();
if (desc == null) return defaultVal;
return desc.getProperty(property, defaultVal);
}
@Override
public Properties readCoreProperties(String coreName) {
if (is50OrLater) {
CoreDescriptorPlus plus = coreDescriptorPlusMap.get(coreName);
if (plus == null) return null;
return new Properties(plus.getCoreDescriptor().getCoreProperties());
} else {
synchronized (coreNodes) {
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
if (coreName.equals(DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null))) {
try {
return readProperties(node);
} catch (XPathExpressionException e) {
return null;
}
}
}
}
}
return null;
}
CoreDescriptorPlus plus = coreDescriptorPlusMap.get(coreName);
if (plus == null) return null;
return new Properties(plus.getCoreDescriptor().getCoreProperties());
@Override
public boolean is50OrLater() { return is50OrLater; }
}
static Properties getCoreProperties(String instanceDir, CoreDescriptor dcore) {
String file = dcore.getPropertiesName();
@ -684,59 +330,10 @@ public class ConfigSolrXml extends Config implements ConfigSolr {
return p;
}
static void addPersistAllCoresStatic(Properties containerProperties, Map<String, String> rootSolrAttribs, Map<String, String> coresAttribs,
File file) {
SolrXMLSerializer.SolrXMLDef solrXMLDef = new SolrXMLSerializer.SolrXMLDef();
solrXMLDef.coresDefs = solrCoreXMLDefs;
solrXMLDef.containerProperties = containerProperties;
solrXMLDef.solrAttribs = rootSolrAttribs;
solrXMLDef.coresAttribs = coresAttribs;
solrXMLSerializer.persistFile(file, solrXMLDef);
@Override
public void substituteProperties() {
config.substituteProperties();
}
static final String DEF_SOLR_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
+ "<solr persistent=\"false\">\n"
+ " <cores adminPath=\"/admin/cores\" defaultCoreName=\""
+ CoreContainer.DEFAULT_DEFAULT_CORE_NAME
+ "\""
+ " host=\"${host:}\" hostPort=\"${hostPort:}\" hostContext=\"${hostContext:}\" zkClientTimeout=\"${zkClientTimeout:15000}\""
+ ">\n"
+ " <core name=\""
+ CoreContainer.DEFAULT_DEFAULT_CORE_NAME
+ "\" shard=\"${shard:}\" collection=\"${collection:}\" instanceDir=\"collection1\" />\n"
+ " </cores>\n" + "</solr>";
}
// It's mightily convenient to have all of the original path names and property values when persisting cores, so
// this little convenience class is just for that.
// Also, let's keep track of anything we added here, especially the instance dir for persistence purposes. We don't
// want, for instance, to persist instanceDir if it was not specified originally.
//
// I suspect that for persistence purposes, we may want to expand this idea to record, say, ${blah}
class CoreDescriptorPlus {
private CoreDescriptor coreDescriptor;
private String filePath;
private Properties propsOrig; // TODO: 5.0. Remove this since it's only really used for persisting.
CoreDescriptorPlus(String filePath, CoreDescriptor descriptor, Properties propsOrig) {
coreDescriptor = descriptor;
this.filePath = filePath;
this.propsOrig = propsOrig;
}
CoreDescriptor getCoreDescriptor() {
return coreDescriptor;
}
String getFilePath() {
return filePath;
}
Properties getPropsOrig() {
return propsOrig;
}
}

358
solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java Normal file
View File

@ -0,0 +1,358 @@
package org.apache.solr.core;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.PropertiesUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
/**
*
*/
public class ConfigSolrXmlOld extends ConfigSolr {
protected static Logger log = LoggerFactory.getLogger(ConfigSolrXmlOld.class);
private NodeList coreNodes = null;
public ConfigSolrXmlOld(Config config, CoreContainer container)
throws ParserConfigurationException, IOException, SAXException {
super(config);
init(container);
}
private void init(CoreContainer container) throws IOException {
// Do sanity checks - we don't want to find new style
// config
failIfFound("solr/str[@name='adminHandler']");
failIfFound("solr/int[@name='coreLoadThreads']");
failIfFound("solr/str[@name='coreRootDirectory']");
failIfFound("solr/solrcloud/int[@name='distribUpdateConnTimeout']");
failIfFound("solr/solrcloud/int[@name='distribUpdateSoTimeout']");
failIfFound("solr/solrcloud/str[@name='host']");
failIfFound("solr/solrcloud/str[@name='hostContext']");
failIfFound("solr/solrcloud/int[@name='hostPort']");
failIfFound("solr/solrcloud/int[@name='leaderVoteWait']");
failIfFound("solr/str[@name='managementPath']");
failIfFound("solr/str[@name='sharedLib']");
failIfFound("solr/str[@name='shareSchema']");
failIfFound("solr/int[@name='transientCacheSize']");
failIfFound("solr/solrcloud/int[@name='zkClientTimeout']");
failIfFound("solr/solrcloud/int[@name='zkHost']");
failIfFound("solr/logging/str[@name='class']");
failIfFound("solr/logging/str[@name='enabled']");
failIfFound("solr/logging/watcher/int[@name='size']");
failIfFound("solr/logging/watcher/int[@name='threshold']");
fillPropMap();
initCoreList(container);
}
private void failIfFound(String xPath) {
if (config.getVal(xPath, false) != null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Should not have found " + xPath +
" solr.xml may be a mix of old and new style formats.");
}
}
private void fillPropMap() {
propMap.put(CfgProp.SOLR_CORELOADTHREADS,
config.getVal("solr/@coreLoadThreads", false));
propMap
.put(CfgProp.SOLR_SHAREDLIB, config.getVal("solr/@sharedLib", false));
propMap.put(CfgProp.SOLR_ZKHOST, config.getVal("solr/@zkHost", false));
propMap.put(CfgProp.SOLR_LOGGING_CLASS,
config.getVal("solr/logging/@class", false));
propMap.put(CfgProp.SOLR_LOGGING_ENABLED,
config.getVal("solr/logging/@enabled", false));
propMap.put(CfgProp.SOLR_LOGGING_WATCHER_SIZE,
config.getVal("solr/logging/watcher/@size", false));
propMap.put(CfgProp.SOLR_LOGGING_WATCHER_THRESHOLD,
config.getVal("solr/logging/watcher/@threshold", false));
propMap.put(CfgProp.SOLR_ADMINHANDLER,
config.getVal("solr/cores/@adminHandler", false));
propMap.put(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT,
config.getVal("solr/cores/@distribUpdateConnTimeout", false));
propMap.put(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT,
config.getVal("solr/cores/@distribUpdateSoTimeout", false));
propMap.put(CfgProp.SOLR_HOST, config.getVal("solr/cores/@host", false));
propMap.put(CfgProp.SOLR_HOSTCONTEXT,
config.getVal("solr/cores/@hostContext", false));
propMap.put(CfgProp.SOLR_HOSTPORT,
config.getVal("solr/cores/@hostPort", false));
propMap.put(CfgProp.SOLR_LEADERVOTEWAIT,
config.getVal("solr/cores/@leaderVoteWait", false));
propMap.put(CfgProp.SOLR_MANAGEMENTPATH,
config.getVal("solr/cores/@managementPath", false));
propMap.put(CfgProp.SOLR_SHARESCHEMA,
config.getVal("solr/cores/@shareSchema", false));
propMap.put(CfgProp.SOLR_TRANSIENTCACHESIZE,
config.getVal("solr/cores/@transientCacheSize", false));
propMap.put(CfgProp.SOLR_ZKCLIENTTIMEOUT,
config.getVal("solr/cores/@zkClientTimeout", false));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_CLASS,
config.getVal("solr/shardHandlerFactory/@class", false));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_NAME,
config.getVal("solr/shardHandlerFactory/@name", false));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_CONNTIMEOUT,
config.getVal("solr/shardHandlerFactory/int[@connTimeout]", false));
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_SOCKETTIMEOUT,
config.getVal("solr/shardHandlerFactory/int[@socketTimeout]", false));
// These have no counterpart in 5.0, asking, for any of these in Solr 5.0
// will result in an error being
// thrown.
propMap.put(CfgProp.SOLR_CORES_DEFAULT_CORE_NAME,
config.getVal("solr/cores/@defaultCoreName", false));
propMap.put(CfgProp.SOLR_PERSISTENT,
config.getVal("solr/@persistent", false));
propMap.put(CfgProp.SOLR_ADMINPATH,
config.getVal("solr/cores/@adminPath", false));
}
private void initCoreList(CoreContainer container) throws IOException {
coreNodes = (NodeList) config.evaluate("solr/cores/core",
XPathConstants.NODESET);
// Check a couple of error conditions
Set<String> names = new HashSet<String>(); // for duplicate names
Map<String,String> dirs = new HashMap<String,String>(); // for duplicate
// data dirs.
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
String name = DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null);
String dataDir = DOMUtil.getAttr(node, CoreDescriptor.CORE_DATADIR, null);
if (name != null) {
if (!names.contains(name)) {
names.add(name);
} else {
String msg = String.format(Locale.ROOT,
"More than one core defined for core named %s", name);
log.error(msg);
}
}
if (dataDir != null) {
if (!dirs.containsKey(dataDir)) {
dirs.put(dataDir, name);
} else {
String msg = String
.format(
Locale.ROOT,
"More than one core points to data dir %s. They are in %s and %s",
dataDir, dirs.get(dataDir), name);
log.warn(msg);
}
}
}
}
@Override
public Map<String, String> readCoreAttributes(String coreName) {
Map<String, String> attrs = new HashMap<String, String>();
synchronized (coreNodes) {
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
if (coreName.equals(DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null))) {
NamedNodeMap attributes = node.getAttributes();
for (int i = 0; i < attributes.getLength(); i++) {
Node attribute = attributes.item(i);
String val = attribute.getNodeValue();
if (CoreDescriptor.CORE_DATADIR.equals(attribute.getNodeName()) ||
CoreDescriptor.CORE_INSTDIR.equals(attribute.getNodeName())) {
if (val.indexOf('$') == -1) {
val = (val != null && !val.endsWith("/")) ? val + '/' : val;
}
}
attrs.put(attribute.getNodeName(), val);
}
return attrs;
}
}
}
return attrs;
}
// Fortunately, we don't iterate over these too often, so the waste is probably tolerable.
@Override
public String getCoreNameFromOrig(String origCoreName,
SolrResourceLoader loader, String coreName) {
// look for an existing node
synchronized (coreNodes) {
// first look for an exact match
Node coreNode = null;
for (int i = 0; i < coreNodes.getLength(); i++) {
Node node = coreNodes.item(i);
String name = DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(name)) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
if (coreNode == null) {
// see if we match with substitution
for (int i = 0; i < coreNodes.getLength(); i++) {
Node node = coreNodes.item(i);
String name = DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(PropertiesUtil.substituteProperty(name,
loader.getCoreProperties()))) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
}
}
return null;
}
@Override
public List<String> getAllCoreNames() {
List<String> ret = new ArrayList<String>();
synchronized (coreNodes) {
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
ret.add(DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null));
}
}
return ret;
}
@Override
public String getProperty(String coreName, String property, String defaultVal) {
synchronized (coreNodes) {
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
if (coreName.equals(DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME,
null))) {
return DOMUtil.getAttr(node, property, defaultVal);
}
}
}
return defaultVal;
}
@Override
public Properties readCoreProperties(String coreName) {
synchronized (coreNodes) {
for (int idx = 0; idx < coreNodes.getLength(); ++idx) {
Node node = coreNodes.item(idx);
if (coreName.equals(DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME,
null))) {
try {
return readProperties(node);
} catch (XPathExpressionException e) {
return null;
}
}
}
}
return null;
}
static Properties getCoreProperties(String instanceDir, CoreDescriptor dcore) {
String file = dcore.getPropertiesName();
if (file == null) file = "conf" + File.separator + "solrcore.properties";
File corePropsFile = new File(file);
if (!corePropsFile.isAbsolute()) {
corePropsFile = new File(instanceDir, file);
}
Properties p = dcore.getCoreProperties();
if (corePropsFile.exists() && corePropsFile.isFile()) {
p = new Properties(dcore.getCoreProperties());
InputStream is = null;
try {
is = new FileInputStream(corePropsFile);
p.load(is);
} catch (IOException e) {
log.warn("Error loading properties ", e);
} finally {
IOUtils.closeQuietly(is);
}
}
return p;
}
public static final String DEF_SOLR_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
+ "<solr persistent=\"false\">\n"
+ " <cores adminPath=\"/admin/cores\" defaultCoreName=\""
+ CoreContainer.DEFAULT_DEFAULT_CORE_NAME
+ "\""
+ " host=\"${host:}\" hostPort=\"${hostPort:}\" hostContext=\"${hostContext:}\" zkClientTimeout=\"${zkClientTimeout:15000}\""
+ ">\n"
+ " <core name=\""
+ CoreContainer.DEFAULT_DEFAULT_CORE_NAME
+ "\" shard=\"${shard:}\" collection=\"${collection:}\" instanceDir=\"collection1\" />\n"
+ " </cores>\n" + "</solr>";
@Override
public void substituteProperties() {
config.substituteProperties();
}
}

765
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -20,25 +20,22 @@ package org.apache.solr.core;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ConcurrentHashMap;
@ -50,9 +47,14 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.dom.DOMSource;
import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.CurrentCoreDescriptorProvider;
import org.apache.solr.cloud.SolrZkServer;
import org.apache.solr.cloud.ZkController;
@ -62,7 +64,6 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
@ -77,10 +78,15 @@ import org.apache.solr.update.SolrCoreState;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.FileUtils;
import org.apache.solr.util.PropertiesUtil;
import org.apache.solr.util.SystemIdResolver;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.impl.StaticLoggerBinder;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
/**
@ -138,6 +144,8 @@ public class CoreContainer
private int distribUpdateSoTimeout = 0;
private int coreLoadThreads;
private CloserThread backgroundCloser = null;
protected volatile ConfigSolr cfg;
private Config origCfg;
{
log.info("New CoreContainer " + System.identityHashCode(this));
@ -173,6 +181,26 @@ public class CoreContainer
public CoreContainer(String solrHome) {
this.solrHome = solrHome;
}
public SolrConfig getSolrConfigFromZk(String zkConfigName, String solrConfigFileName,
SolrResourceLoader resourceLoader) {
SolrConfig cfg = null;
try {
byte[] config = zkController.getConfigFileData(zkConfigName,
solrConfigFileName);
InputSource is = new InputSource(new ByteArrayInputStream(config));
is.setSystemId(SystemIdResolver
.createSystemIdFromResourceName(solrConfigFileName));
cfg = solrConfigFileName == null ? new SolrConfig(resourceLoader,
SolrConfig.DEFAULT_CONF_FILE, is) : new SolrConfig(resourceLoader,
solrConfigFileName, is);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"getSolrConfigFromZK failed for " + zkConfigName + " "
+ solrConfigFileName, e);
}
return cfg;
}
protected void initZooKeeper(String zkHost, int zkClientTimeout) {
// if zkHost sys property is not set, we are not using ZooKeeper
@ -326,9 +354,10 @@ public class CoreContainer
if (fconf.exists()) {
cores.load(solrHome, fconf);
} else {
// Back compart support
log.info("no solr.xml found. using default old-style solr.xml");
try {
cores.load(solrHome, new ByteArrayInputStream(ConfigSolrXml.DEF_SOLR_XML.getBytes("UTF-8")), null);
cores.load(solrHome, new ByteArrayInputStream(ConfigSolrXmlOld.DEF_SOLR_XML.getBytes("UTF-8")), null);
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"CoreContainer.Initialize failed when trying to load default solr.xml file", e);
@ -381,12 +410,19 @@ public class CoreContainer
this.loader = new SolrResourceLoader(dir);
solrHome = loader.getInstanceDir();
ConfigSolr cfg;
//ConfigSolr cfg;
// keep orig config for persist to consult. TODO: Remove this silly stuff for 5.0, persistence not supported.
try {
cfg = new ConfigSolrXml(loader, null, is, null, false, this);
this.cfg = new ConfigSolrXml(loader, (ConfigSolrXml) cfg, this);
Config config = new Config(loader, null, new InputSource(is), null, false);
this.origCfg = new Config(loader, null, copyDoc(config.getDocument()));
boolean oldStyle = (config.getNode("solr/cores", false) != null);
if (oldStyle) {
this.cfg = new ConfigSolrXmlOld(config, this);
} else {
this.cfg = new ConfigSolrXml(config, this);
}
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "", e);
}
@ -403,7 +439,7 @@ public class CoreContainer
loader.reloadLuceneSPI();
}
shardHandlerFactory = cfg.initShardHandler();
shardHandlerFactory = initShardHandler(cfg);
coreMaps.allocateLazyCores(cfg, loader);
@ -453,7 +489,7 @@ public class CoreContainer
}
if (! cfg.is50OrLater()) { //TODO: Remove for 5.0
if (cfg instanceof ConfigSolrXmlOld) { //TODO: Remove for 5.0
String dcoreName = cfg.get(ConfigSolr.CfgProp.SOLR_CORES_DEFAULT_CORE_NAME, null);
if (dcoreName != null && !dcoreName.isEmpty()) {
defaultCoreName = dcoreName;
@ -651,11 +687,36 @@ public class CoreContainer
}
}
}
private ShardHandlerFactory initShardHandler(ConfigSolr configSolr) {
PluginInfo info = null;
Node shfn = configSolr.getConfig().getNode("solr/cores/shardHandlerFactory", false);
if (shfn != null) {
info = new PluginInfo(shfn, "shardHandlerFactory", false, true);
} else {
Map m = new HashMap();
m.put("class", HttpShardHandlerFactory.class.getName());
info = new PluginInfo("shardHandlerFactory", m, null, Collections.<PluginInfo>emptyList());
}
ShardHandlerFactory fac;
try {
fac = configSolr.getConfig().getResourceLoader().findClass(info.className, ShardHandlerFactory.class).newInstance();
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error instantiating shardHandlerFactory class " + info.className);
}
if (fac instanceof PluginInfoInitialized) {
((PluginInfoInitialized) fac).init(info);
}
return fac;
}
// To make this available to TestHarness.
protected void initShardHandler() {
if (cfg != null) {
cfg.initShardHandler();
initShardHandler(cfg);
} else {
// Cough! Hack! But tests run this way.
HttpShardHandlerFactory fac = new HttpShardHandlerFactory();
@ -664,8 +725,6 @@ public class CoreContainer
}
private volatile boolean isShutDown = false;
volatile ConfigSolr cfg;
public boolean isShutDown() {
return isShutDown;
@ -957,54 +1016,6 @@ public class CoreContainer
}
/**
* Removes all references to the oldSchema from the schema cache; places an entry
* in the schema cache for the newSchema; and replaces references to the oldSchema
* with references to the newSchema in all registered cores.
*
* @param initiatingCore The initiating core; schema references doesn't need to be changed here
* @param oldSchema The schema to remove references to
* @param newSchema The schema to add references to
*/
public void replaceSchema(SolrCore initiatingCore, IndexSchema oldSchema, IndexSchema newSchema) {
if (null != indexSchemaCache) { // isShareSchema() == true
// Remove references to the oldSchema from the schema cache
for (Iterator<Map.Entry<String,IndexSchema>> iter = indexSchemaCache.entrySet().iterator() ; iter.hasNext() ; ) {
Map.Entry<String,IndexSchema> entry = iter.next();
if (oldSchema == entry.getValue()) {
iter.remove();
}
}
// Cache the new schema
final String newSchemaResourceName
= IndexSchemaFactory.getResourceNameToBeUsed(newSchema.getResourceName(), initiatingCore.getSolrConfig());
File schemaFile = new File(newSchemaResourceName);
if ( ! schemaFile.isAbsolute()) {
schemaFile = new File(initiatingCore.getResourceLoader().getConfigDir(), schemaFile.getPath());
}
if (schemaFile.exists()) {
String key = schemaFile.getAbsolutePath()
+ ":"
+ new SimpleDateFormat("yyyyMMddHHmmss", Locale.ROOT).format(new Date(
schemaFile.lastModified()));
indexSchemaCache.put(key, newSchema);
}
// Replace oldSchema references with newSchema references in all active cores
for (String coreName : coreMaps.getAllCoreNames()) {
SolrCore activeCore = coreMaps.getCoreFromAnyList(coreName);
if (null != activeCore) {
if (initiatingCore != activeCore) {
if (oldSchema == activeCore.getLatestSchema()) {
activeCore.setLatestSchema(newSchema);
}
}
}
}
}
}
/**
* Creates a new core based on a descriptor but does not register it.
*
* @param dcore a core descriptor
@ -1314,6 +1325,7 @@ public class CoreContainer
// all of the following properties aren't synchronized
// but this should be OK since they normally won't be changed rapidly
@Deprecated
public boolean isPersistent() {
return persistent;
}
@ -1378,7 +1390,7 @@ public class CoreContainer
/** Persists the cores config file in cores.xml. */
public void persist() {
persistFile(null);
persistFile(configFile);
}
/**
@ -1395,7 +1407,9 @@ public class CoreContainer
/** Persists the cores config file in a user provided file. */
//TODO: obsolete in SOLR 5.0
public void persistFile(File file) {
if (cfg != null && cfg.is50OrLater()) return;
assert file != null;
// only the old solrxml persists
if (cfg != null && !(cfg instanceof ConfigSolrXmlOld)) return;
log.info("Persisting cores config to " + (file == null ? configFile : file));
@ -1434,7 +1448,11 @@ public class CoreContainer
Integer.toString(this.transientCacheSize), Integer.toString(Integer.MAX_VALUE));
}
coreMaps.persistCores(cfg, containerProperties, rootSolrAttribs, coresAttribs, file, configFile, loader);
try {
coreMaps.persistCores(origCfg, containerProperties, rootSolrAttribs, coresAttribs, file, configFile, loader);
} catch (XPathExpressionException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, null, e);
}
}
private String intToString(Integer integer) {
@ -1452,7 +1470,7 @@ public class CoreContainer
if (attribValue != null) {
String origValue = cfg.getOrigProp(prop, null);
if (origValue == null && defaultValue != null && attribValue.equals(defaultValue)) return;
if (attribValue.equals(PropertiesUtil.substituteProperty(origValue, loader.getCoreProperties()))) {
@ -1484,11 +1502,6 @@ public class CoreContainer
return shardHandlerFactory;
}
private SolrConfig getSolrConfigFromZk(String zkConfigName, String solrConfigFileName,
SolrResourceLoader resourceLoader)
{
return cfg.getSolrConfigFromZk(zkController, zkConfigName, solrConfigFileName, resourceLoader);
}
// Just to tidy up the code where it did this in-line.
private SolrException recordAndThrow(String name, String msg, Exception ex) {
synchronized (coreInitFailures) {
@ -1498,6 +1511,7 @@ public class CoreContainer
log.error(msg, ex);
return new SolrException(ErrorCode.SERVER_ERROR, msg, ex);
}
String getCoreToOrigName(SolrCore core) {
return coreMaps.getCoreToOrigName(core);
}
@ -1505,600 +1519,19 @@ public class CoreContainer
public String getBadCoreMessage(String name) {
return cfg.getBadConfigCoreMessage(name);
}
private Document copyDoc(Document document) throws TransformerException {
TransformerFactory tfactory = TransformerFactory.newInstance();
Transformer tx = tfactory.newTransformer();
DOMSource source = new DOMSource(document);
DOMResult result = new DOMResult();
tx.transform(source,result);
return (Document)result.getNode();
}
}
// Introducing the two new maps (transientCores and dynamicDescriptors) introduced some locking complexities. Rather
// than try to keep them all straight in the code, use this class you need to access any of:
// cores
// transientCores
// dynamicDescriptors
//
class CoreMaps {
private static Object locker = new Object(); // for locking around manipulating any of the core maps.
private final Map<String, SolrCore> cores = new LinkedHashMap<String, SolrCore>(); // For "permanent" cores
//WARNING! The _only_ place you put anything into the list of transient cores is with the putTransientCore method!
private Map<String, SolrCore> transientCores = new LinkedHashMap<String, SolrCore>(); // For "lazily loaded" cores
private final Map<String, CoreDescriptor> dynamicDescriptors = new LinkedHashMap<String, CoreDescriptor>();
private final Map<String, SolrCore> createdCores = new LinkedHashMap<String, SolrCore>();
private Map<SolrCore, String> coreToOrigName = new ConcurrentHashMap<SolrCore, String>();
private final CoreContainer container;
// This map will hold objects that are being currently operated on. The core (value) may be null in the case of
// initial load. The rule is, never to any operation on a core that is currently being operated upon.
private static final Set<String> pendingCoreOps = new HashSet<String>();
// Due to the fact that closes happen potentially whenever anything is _added_ to the transient core list, we need
// to essentially queue them up to be handled via pendingCoreOps.
private static final List<SolrCore> pendingCloses = new ArrayList<SolrCore>();
CoreMaps(CoreContainer container) {
this.container = container;
}
// Trivial helper method for load, note it implements LRU on transient cores. Also note, if
// there is no setting for max size, nothing is done and all cores go in the regular "cores" list
protected void allocateLazyCores(final ConfigSolr cfg, final SolrResourceLoader loader) {
final int transientCacheSize = cfg.getInt(ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, Integer.MAX_VALUE);
if (transientCacheSize != Integer.MAX_VALUE) {
CoreContainer.log.info("Allocating transient cache for {} transient cores", transientCacheSize);
transientCores = new LinkedHashMap<String, SolrCore>(transientCacheSize, 0.75f, true) {
@Override
protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
if (size() > transientCacheSize) {
synchronized (locker) {
pendingCloses.add(eldest.getValue()); // Essentially just queue this core up for closing.
locker.notifyAll(); // Wakes up closer thread too
}
return true;
}
return false;
}
};
}
}
protected void putDynamicDescriptor(String rawName, CoreDescriptor p) {
synchronized (locker) {
dynamicDescriptors.put(rawName, p);
}
}
// We are shutting down. You can't hold the lock on the various lists of cores while they shut down, so we need to
// make a temporary copy of the names and shut them down outside the lock.
protected void clearMaps(ConfigSolr cfg) {
List<String> coreNames;
List<String> transientNames;
List<SolrCore> pendingToClose;
// It might be possible for one of the cores to move from one list to another while we're closing them. So
// loop through the lists until they're all empty. In particular, the core could have moved from the transient
// list to the pendingCloses list.
while (true) {
synchronized (locker) {
coreNames = new ArrayList(cores.keySet());
transientNames = new ArrayList(transientCores.keySet());
pendingToClose = new ArrayList(pendingCloses);
}
if (coreNames.size() == 0 && transientNames.size() == 0 && pendingToClose.size() == 0) break;
for (String coreName : coreNames) {
SolrCore core = cores.get(coreName);
if (core == null) {
CoreContainer.log.info("Core " + coreName + " moved from core container list before closing.");
} else {
try {
addPersistOneCore(cfg, container.loader, core.getCoreDescriptor(), getCoreToOrigName(core));
core.close();
} catch (Throwable t) {
SolrException.log(CoreContainer.log, "Error shutting down core", t);
} finally {
synchronized (locker) {
cores.remove(coreName);
}
}
}
}
for (String coreName : transientNames) {
SolrCore core = transientCores.get(coreName);
if (core == null) {
CoreContainer.log.info("Core " + coreName + " moved from transient core container list before closing.");
} else {
try {
core.close();
} catch (Throwable t) {
SolrException.log(CoreContainer.log, "Error shutting down core", t);
} finally {
synchronized (locker) {
transientCores.remove(coreName);
}
}
}
}
// We might have some cores that we were _thinking_ about shutting down, so take care of those too.
for (SolrCore core : pendingToClose) {
try {
core.close();
} catch (Throwable t) {
SolrException.log(CoreContainer.log, "Error shutting down core", t);
} finally {
synchronized (locker) {
pendingCloses.remove(core);
}
}
}
}
}
protected void addCoresToList(ArrayList<SolrCoreState> coreStates) {
List<SolrCore> addCores;
synchronized (locker) {
addCores = new ArrayList<SolrCore>(cores.values());
}
for (SolrCore core : addCores) {
coreStates.add(core.getUpdateHandler().getSolrCoreState());
}
}
//WARNING! This should be the _only_ place you put anything into the list of transient cores!
protected SolrCore putTransientCore(ConfigSolr cfg, String name, SolrCore core, SolrResourceLoader loader) {
SolrCore retCore;
CoreContainer.log.info("Opening transient core {}", name);
synchronized (locker) {
retCore = transientCores.put(name, core);
}
return retCore;
}
protected SolrCore putCore(String name, SolrCore core) {
synchronized (locker) {
return cores.put(name, core);
}
}
List<SolrCore> getCores() {
List<SolrCore> lst = new ArrayList<SolrCore>();
synchronized (locker) {
lst.addAll(cores.values());
return lst;
}
}
Set<String> getCoreNames() {
Set<String> set = new TreeSet<String>();
synchronized (locker) {
set.addAll(cores.keySet());
set.addAll(transientCores.keySet());
}
return set;
}
List<String> getCoreNames(SolrCore core) {
List<String> lst = new ArrayList<String>();
synchronized (locker) {
for (Map.Entry<String, SolrCore> entry : cores.entrySet()) {
if (core == entry.getValue()) {
lst.add(entry.getKey());
}
}
for (Map.Entry<String, SolrCore> entry : transientCores.entrySet()) {
if (core == entry.getValue()) {
lst.add(entry.getKey());
}
}
}
return lst;
}
/**
* Gets a list of all cores, loaded and unloaded (dynamic)
*
* @return all cores names, whether loaded or unloaded.
*/
public Collection<String> getAllCoreNames() {
Set<String> set = new TreeSet<String>();
synchronized (locker) {
set.addAll(cores.keySet());
set.addAll(transientCores.keySet());
set.addAll(dynamicDescriptors.keySet());
set.addAll(createdCores.keySet());
}
return set;
}
SolrCore getCore(String name) {
synchronized (locker) {
return cores.get(name);
}
}
protected void swap(String n0, String n1) {
synchronized (locker) {
SolrCore c0 = cores.get(n0);
SolrCore c1 = cores.get(n1);
if (c0 == null)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No such core: " + n0);
if (c1 == null)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No such core: " + n1);
cores.put(n0, c1);
cores.put(n1, c0);
c0.setName(n1);
c0.getCoreDescriptor().putProperty(CoreDescriptor.CORE_NAME, n1);
c1.setName(n0);
c1.getCoreDescriptor().putProperty(CoreDescriptor.CORE_NAME, n0);
}
}
protected SolrCore remove(String name, boolean removeOrig) {
synchronized (locker) {
SolrCore tmp = cores.remove(name);
SolrCore ret = null;
if (removeOrig && tmp != null) {
coreToOrigName.remove(tmp);
}
ret = (ret == null) ? tmp : ret;
// It could have been a newly-created core. It could have been a transient core. The newly-created cores
// in particular should be checked. It could have been a dynamic core.
tmp = transientCores.remove(name);
ret = (ret == null) ? tmp : ret;
tmp = createdCores.remove(name);
ret = (ret == null) ? tmp : ret;
dynamicDescriptors.remove(name);
return ret;
}
}
protected void putCoreToOrigName(SolrCore c, String name) {
synchronized (locker) {
coreToOrigName.put(c, name);
}
}
protected void removeCoreToOrigName(SolrCore newCore, SolrCore core) {
synchronized (locker) {
String origName = coreToOrigName.remove(core);
if (origName != null) {
coreToOrigName.put(newCore, origName);
}
}
}
public SolrCore getCoreFromAnyList(String name) {
SolrCore core;
synchronized (locker) {
core = cores.get(name);
if (core != null) {
return core;
}
if (dynamicDescriptors.size() == 0) {
return null; // Nobody even tried to define any transient cores, so we're done.
}
// Now look for already loaded transient cores.
return transientCores.get(name);
}
}
protected CoreDescriptor getDynamicDescriptor(String name) {
synchronized (locker) {
return dynamicDescriptors.get(name);
}
}
protected boolean isLoaded(String name) {
synchronized (locker) {
if (cores.containsKey(name)) {
return true;
}
if (transientCores.containsKey(name)) {
return true;
}
}
return false;
}
protected CoreDescriptor getUnloadedCoreDescriptor(String cname) {
synchronized (locker) {
CoreDescriptor desc = dynamicDescriptors.get(cname);
if (desc == null) {
return null;
}
return new CoreDescriptor(desc);
}
}
protected String getCoreToOrigName(SolrCore solrCore) {
synchronized (locker) {
return coreToOrigName.get(solrCore);
}
}
protected void publishCoresAsDown(ZkController zkController) {
synchronized (locker) {
for (SolrCore core : cores.values()) {
try {
zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
} catch (KeeperException e) {
CoreContainer.log.error("", e);
} catch (InterruptedException e) {
CoreContainer.log.error("", e);
}
}
for (SolrCore core : transientCores.values()) {
try {
zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
} catch (KeeperException e) {
CoreContainer.log.error("", e);
} catch (InterruptedException e) {
CoreContainer.log.error("", e);
}
}
}
}
// Irrepressably ugly bit of the transition in SOLR-4196, but there as at least one test case that follows
// this path, presumably it's there for a reason.
// This is really perverse, but all we need the here is to call a couple of static methods that for back-compat
// purposes
public void persistCores(ConfigSolr cfg, Properties containerProperties, Map<String, String> rootSolrAttribs,
Map<String, String> coresAttribs, File file, File configFile, SolrResourceLoader loader) {
// This is expensive in the maximal case, but I think necessary. It should keep a reference open to all of the
// current cores while they are saved. Remember that especially the transient core can come and go.
//
// TODO: 5.0. remove the possibility of storing core descirptors in solr.xml?
//
synchronized (locker) {
if (cfg == null) {
ConfigSolrXml.initPersistStatic();
persistCores(cfg, cores, loader);
persistCores(cfg, transientCores, loader);
// add back all the cores that aren't loaded, either in cores or transient cores
for (Map.Entry<String, CoreDescriptor> ent : dynamicDescriptors.entrySet()) {
if (! cores.containsKey(ent.getKey()) && ! transientCores.containsKey(ent.getKey())) {
addPersistOneCore(cfg, loader, ent.getValue(), null);
}
}
for (Map.Entry<String, SolrCore> ent : createdCores.entrySet()) {
if (! cores.containsKey(ent.getKey()) && ! transientCores.containsKey(ent.getKey())
&& ! dynamicDescriptors.containsKey(ent.getKey())) {
addPersistOneCore(cfg, loader, ent.getValue().getCoreDescriptor(), null);
}
}
ConfigSolrXml.addPersistAllCoresStatic(containerProperties, rootSolrAttribs, coresAttribs,
(file == null ? configFile : file));
} else {
cfg.initPersist();
persistCores(cfg, cores, loader);
persistCores(cfg, transientCores, loader);
// add back all the cores that aren't loaded, either in cores or transient cores
for (Map.Entry<String, CoreDescriptor> ent : dynamicDescriptors.entrySet()) {
if (! cores.containsKey(ent.getKey()) && ! transientCores.containsKey(ent.getKey())) {
addPersistOneCore(cfg, loader, ent.getValue(), null);
}
}
for (Map.Entry<String, SolrCore> ent : createdCores.entrySet()) {
if (! cores.containsKey(ent.getKey()) && ! transientCores.containsKey(ent.getKey())
&& ! dynamicDescriptors.containsKey(ent.getKey())) {
addPersistOneCore(cfg, loader, ent.getValue().getCoreDescriptor(), null);
}
}
cfg.addPersistAllCores(containerProperties, rootSolrAttribs, coresAttribs, (file == null ? configFile : file));
}
}
}
// Wait here until any pending operations (load, unload or reload) are completed on this core.
protected SolrCore waitAddPendingCoreOps(String name) {
// Keep multiple threads from operating on a core at one time.
synchronized (locker) {
boolean pending;
do { // Are we currently doing anything to this core? Loading, unloading, reloading?
pending = pendingCoreOps.contains(name); // wait for the core to be done being operated upon
if (! pending) { // Linear list, but shouldn't be too long
for (SolrCore core : pendingCloses) {
if (core.getName().equals(name)) {
pending = true;
break;
}
}
}
if (container.isShutDown()) return null; // Just stop already.
if (pending) {
try {
locker.wait();
} catch (InterruptedException e) {
return null; // Seems best not to do anything at all if the thread is interrupted
}
}
} while (pending);
// We _really_ need to do this within the synchronized block!
if (! container.isShutDown()) {
if (! pendingCoreOps.add(name)) {
CoreContainer.log.warn("Replaced an entry in pendingCoreOps {}, we should not be doing this", name);
}
return getCoreFromAnyList(name); // we might have been _unloading_ the core, so return the core if it was loaded.
}
}
return null;
}
// We should always be removing the first thing in the list with our name! The idea here is to NOT do anything n
// any core while some other operation is working on that core.
protected void removeFromPendingOps(String name) {
synchronized (locker) {
if (! pendingCoreOps.remove(name)) {
CoreContainer.log.warn("Tried to remove core {} from pendingCoreOps and it wasn't there. ", name);
}
locker.notifyAll();
}
}
protected void persistCores(ConfigSolr cfg, Map<String, SolrCore> whichCores, SolrResourceLoader loader) {
for (SolrCore solrCore : whichCores.values()) {
addPersistOneCore(cfg, loader, solrCore.getCoreDescriptor(), getCoreToOrigName(solrCore));
}
}
private void addIfNotNull(Map<String, String> coreAttribs, String key, String value) {
if (value == null) return;
coreAttribs.put(key, value);
}
protected void addPersistOneCore(ConfigSolr cfg, SolrResourceLoader loader, CoreDescriptor dcore, String origCoreName) {
String coreName = dcore.getProperty(CoreDescriptor.CORE_NAME);
Map<String, String> coreAttribs = new HashMap<String, String>();
Properties persistProps = new Properties();
CloudDescriptor cd = dcore.getCloudDescriptor();
String collection = null;
if (cd != null) collection = cd.getCollectionName();
String instDir = dcore.getRawInstanceDir();
if (cfg == null) {
addIfNotNull(coreAttribs, CoreDescriptor.CORE_NAME, coreName);
addIfNotNull(coreAttribs, CoreDescriptor.CORE_CONFIG, dcore.getDefaultConfigName());
addIfNotNull(coreAttribs, CoreDescriptor.CORE_SCHEMA, dcore.getDefaultSchemaName());
addIfNotNull(coreAttribs, CoreDescriptor.CORE_DATADIR, dcore.getProperty(CoreDescriptor.CORE_DATADIR));
addIfNotNull(coreAttribs, CoreDescriptor.CORE_ULOGDIR, dcore.getProperty(CoreDescriptor.CORE_ULOGDIR));
addIfNotNull(coreAttribs, CoreDescriptor.CORE_TRANSIENT, dcore.getProperty(CoreDescriptor.CORE_TRANSIENT));
addIfNotNull(coreAttribs, CoreDescriptor.CORE_LOADONSTARTUP, dcore.getProperty(CoreDescriptor.CORE_LOADONSTARTUP));
// we don't try and preserve sys prop defs in these
addIfNotNull(coreAttribs, CoreDescriptor.CORE_PROPERTIES, dcore.getPropertiesName());
// Add in any non-standard bits of data
Set<String> std = new TreeSet<String>();
Properties allProps = dcore.getCoreProperties();
std.addAll(Arrays.asList(CoreDescriptor.standardPropNames));
for (String prop : allProps.stringPropertyNames()) {
if (! std.contains(prop)) {
persistProps.put(prop, dcore.getProperty(prop));
}
}
if (StringUtils.isNotBlank(collection) && !collection.equals(coreName)) {
coreAttribs.put(CoreDescriptor.CORE_COLLECTION, collection);
}
} else {
if (origCoreName == null) {
origCoreName = coreName;
}
String tmp = cfg.getCoreNameFromOrig(origCoreName, loader, coreName);
if (tmp != null) coreName = tmp;
coreAttribs = cfg.readCoreAttributes(origCoreName);
persistProps = cfg.readCoreProperties(origCoreName);
coreAttribs.put(CoreDescriptor.CORE_NAME, coreName);
if (coreAttribs.containsKey(CoreDescriptor.CORE_COLLECTION)) collection = coreAttribs
.get(CoreDescriptor.CORE_COLLECTION);
if (coreAttribs.containsKey(CoreDescriptor.CORE_INSTDIR)) instDir = coreAttribs
.get(CoreDescriptor.CORE_INSTDIR);
addIfNotNull(coreAttribs, CoreDescriptor.CORE_INSTDIR,
dcore.getRawInstanceDir());
coreAttribs.put(CoreDescriptor.CORE_COLLECTION,
StringUtils.isNotBlank(collection) ? collection : dcore.getName());
}
// Default value here is same as old code.
addIfNotNull(coreAttribs, CoreDescriptor.CORE_INSTDIR, instDir);
// Emulating the old code, just overwrite shard and roles if present in the cloud descriptor
if (cd != null) {
addIfNotNull(coreAttribs, CoreDescriptor.CORE_SHARD, cd.getShardId());
addIfNotNull(coreAttribs, CoreDescriptor.CORE_ROLES, cd.getRoles());
}
coreAttribs.put(CoreDescriptor.CORE_LOADONSTARTUP, Boolean.toString(dcore.isLoadOnStartup()));
coreAttribs.put(CoreDescriptor.CORE_TRANSIENT, Boolean.toString(dcore.isTransient()));
if (cfg != null) {
cfg.addPersistCore(coreName, persistProps, coreAttribs);
} else {
// Another awkward bit for back-compat for SOLR-4196
ConfigSolrXml.addPersistCore(persistProps, coreAttribs);
}
}
protected Object getLocker() { return locker; }
// Be a little careful. We don't want to either open or close a core unless it's _not_ being opened or closed by
// another thread. So within this lock we'll walk along the list of pending closes until we find something NOT in
// the list of threads currently being loaded or reloaded. The "usual" case will probably return the very first
// one anyway..
protected SolrCore getCoreToClose() {
synchronized (locker) {
for (SolrCore core : pendingCloses) {
if (! pendingCoreOps.contains(core.getName())) {
pendingCoreOps.add(core.getName());
pendingCloses.remove(core);
return core;
}
}
}
return null;
}
protected void addCreated(SolrCore core) {
synchronized (locker) {
createdCores.put(core.getName(), core);
}
}
protected String checkUniqueDataDir(String targetPath) {
// Have to check
// loaded cores
// transient cores
// dynamic cores
synchronized (locker) {
for (SolrCore core : cores.values()) {
if (targetPath.equals(core.getDataDir())) return core.getName();
}
for (SolrCore core : transientCores.values()) {
if (targetPath.equals(core.getDataDir())) return core.getName();
}
for (CoreDescriptor desc : dynamicDescriptors.values()) {
if (targetPath.equals(desc.getDataDir())) return desc.getName();
}
}
return null;
}
}
class CloserThread extends Thread {
CoreContainer container;
@ -2131,8 +1564,9 @@ class CloserThread extends Thread {
removeMe != null && !container.isShutDown();
removeMe = coreMaps.getCoreToClose()) {
try {
coreMaps.addPersistOneCore(cfg, container.loader, removeMe.getCoreDescriptor(),
container.getCoreToOrigName(removeMe));
// nocommit: wtf is this?
// coreMaps.addPersistOneCore(cfg, container.loader, removeMe.getCoreDescriptor(),
// container.getCoreToOrigName(removeMe), null);
removeMe.close();
} finally {
coreMaps.removeFromPendingOps(removeMe.getName());
@ -2140,4 +1574,5 @@ class CloserThread extends Thread {
}
}
}
}

614
solr/core/src/java/org/apache/solr/core/CoreMaps.java Normal file
View File

@ -0,0 +1,614 @@
package org.apache.solr.core;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.SolrXMLSerializer.SolrCoreXMLDef;
import org.apache.solr.update.SolrCoreState;
import org.apache.solr.util.DOMUtil;
import org.apache.zookeeper.KeeperException;
import org.w3c.dom.Node;
class CoreMaps {
private static SolrXMLSerializer SOLR_XML_SERIALIZER = new SolrXMLSerializer();
private static Object locker = new Object(); // for locking around manipulating any of the core maps.
private final Map<String, SolrCore> cores = new LinkedHashMap<String, SolrCore>(); // For "permanent" cores
//WARNING! The _only_ place you put anything into the list of transient cores is with the putTransientCore method!
private Map<String, SolrCore> transientCores = new LinkedHashMap<String, SolrCore>(); // For "lazily loaded" cores
private final Map<String, CoreDescriptor> dynamicDescriptors = new LinkedHashMap<String, CoreDescriptor>();
private final Map<String, SolrCore> createdCores = new LinkedHashMap<String, SolrCore>();
private Map<SolrCore, String> coreToOrigName = new ConcurrentHashMap<SolrCore, String>();
private final CoreContainer container;
// This map will hold objects that are being currently operated on. The core (value) may be null in the case of
// initial load. The rule is, never to any operation on a core that is currently being operated upon.
private static final Set<String> pendingCoreOps = new HashSet<String>();
// Due to the fact that closes happen potentially whenever anything is _added_ to the transient core list, we need
// to essentially queue them up to be handled via pendingCoreOps.
private static final List<SolrCore> pendingCloses = new ArrayList<SolrCore>();
CoreMaps(CoreContainer container) {
this.container = container;
}
// Trivial helper method for load, note it implements LRU on transient cores. Also note, if
// there is no setting for max size, nothing is done and all cores go in the regular "cores" list
protected void allocateLazyCores(final ConfigSolr cfg, final SolrResourceLoader loader) {
final int transientCacheSize = cfg.getInt(ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, Integer.MAX_VALUE);
if (transientCacheSize != Integer.MAX_VALUE) {
CoreContainer.log.info("Allocating transient cache for {} transient cores", transientCacheSize);
transientCores = new LinkedHashMap<String, SolrCore>(transientCacheSize, 0.75f, true) {
@Override
protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
if (size() > transientCacheSize) {
synchronized (locker) {
pendingCloses.add(eldest.getValue()); // Essentially just queue this core up for closing.
locker.notifyAll(); // Wakes up closer thread too
}
return true;
}
return false;
}
};
}
}
protected void putDynamicDescriptor(String rawName, CoreDescriptor p) {
synchronized (locker) {
dynamicDescriptors.put(rawName, p);
}
}
// We are shutting down. You can't hold the lock on the various lists of cores while they shut down, so we need to
// make a temporary copy of the names and shut them down outside the lock.
protected void clearMaps(ConfigSolr cfg) {
List<String> coreNames;
List<String> transientNames;
List<SolrCore> pendingToClose;
// It might be possible for one of the cores to move from one list to another while we're closing them. So
// loop through the lists until they're all empty. In particular, the core could have moved from the transient
// list to the pendingCloses list.
while (true) {
synchronized (locker) {
coreNames = new ArrayList<String>(cores.keySet());
transientNames = new ArrayList<String>(transientCores.keySet());
pendingToClose = new ArrayList<SolrCore>(pendingCloses);
}
if (coreNames.size() == 0 && transientNames.size() == 0 && pendingToClose.size() == 0) break;
for (String coreName : coreNames) {
SolrCore core = cores.get(coreName);
if (core == null) {
CoreContainer.log.info("Core " + coreName + " moved from core container list before closing.");
} else {
try {
// nocommit: wtf is this?
// addPersistOneCore(cfg, container.loader, core.getCoreDescriptor(), getCoreToOrigName(core));
core.close();
} catch (Throwable t) {
SolrException.log(CoreContainer.log, "Error shutting down core", t);
} finally {
synchronized (locker) {
cores.remove(coreName);
}
}
}
}
for (String coreName : transientNames) {
SolrCore core = transientCores.get(coreName);
if (core == null) {
CoreContainer.log.info("Core " + coreName + " moved from transient core container list before closing.");
} else {
try {
core.close();
} catch (Throwable t) {
SolrException.log(CoreContainer.log, "Error shutting down core", t);
} finally {
synchronized (locker) {
transientCores.remove(coreName);
}
}
}
}
// We might have some cores that we were _thinking_ about shutting down, so take care of those too.
for (SolrCore core : pendingToClose) {
try {
core.close();
} catch (Throwable t) {
SolrException.log(CoreContainer.log, "Error shutting down core", t);
} finally {
synchronized (locker) {
pendingCloses.remove(core);
}
}
}
}
}
protected void addCoresToList(ArrayList<SolrCoreState> coreStates) {
List<SolrCore> addCores;
synchronized (locker) {
addCores = new ArrayList<SolrCore>(cores.values());
}
for (SolrCore core : addCores) {
coreStates.add(core.getUpdateHandler().getSolrCoreState());
}
}
//WARNING! This should be the _only_ place you put anything into the list of transient cores!
protected SolrCore putTransientCore(ConfigSolr cfg, String name, SolrCore core, SolrResourceLoader loader) {
SolrCore retCore;
CoreContainer.log.info("Opening transient core {}", name);
synchronized (locker) {
retCore = transientCores.put(name, core);
}
return retCore;
}
protected SolrCore putCore(String name, SolrCore core) {
synchronized (locker) {
return cores.put(name, core);
}
}
List<SolrCore> getCores() {
List<SolrCore> lst = new ArrayList<SolrCore>();
synchronized (locker) {
lst.addAll(cores.values());
return lst;
}
}
Set<String> getCoreNames() {
Set<String> set = new TreeSet<String>();
synchronized (locker) {
set.addAll(cores.keySet());
set.addAll(transientCores.keySet());
}
return set;
}
List<String> getCoreNames(SolrCore core) {
List<String> lst = new ArrayList<String>();
synchronized (locker) {
for (Map.Entry<String, SolrCore> entry : cores.entrySet()) {
if (core == entry.getValue()) {
lst.add(entry.getKey());
}
}
for (Map.Entry<String, SolrCore> entry : transientCores.entrySet()) {
if (core == entry.getValue()) {
lst.add(entry.getKey());
}
}
}
return lst;
}
/**
* Gets a list of all cores, loaded and unloaded (dynamic)
*
* @return all cores names, whether loaded or unloaded.
*/
public Collection<String> getAllCoreNames() {
Set<String> set = new TreeSet<String>();
synchronized (locker) {
set.addAll(cores.keySet());
set.addAll(transientCores.keySet());
set.addAll(dynamicDescriptors.keySet());
set.addAll(createdCores.keySet());
}
return set;
}
SolrCore getCore(String name) {
synchronized (locker) {
return cores.get(name);
}
}
protected void swap(String n0, String n1) {
synchronized (locker) {
SolrCore c0 = cores.get(n0);
SolrCore c1 = cores.get(n1);
if (c0 == null)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No such core: " + n0);
if (c1 == null)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No such core: " + n1);
cores.put(n0, c1);
cores.put(n1, c0);
c0.setName(n1);
c0.getCoreDescriptor().putProperty(CoreDescriptor.CORE_NAME, n1);
c1.setName(n0);
c1.getCoreDescriptor().putProperty(CoreDescriptor.CORE_NAME, n0);
}
}
protected SolrCore remove(String name, boolean removeOrig) {
synchronized (locker) {
SolrCore tmp = cores.remove(name);
SolrCore ret = null;
if (removeOrig && tmp != null) {
coreToOrigName.remove(tmp);
}
ret = (ret == null) ? tmp : ret;
// It could have been a newly-created core. It could have been a transient core. The newly-created cores
// in particular should be checked. It could have been a dynamic core.
tmp = transientCores.remove(name);
ret = (ret == null) ? tmp : ret;
tmp = createdCores.remove(name);
ret = (ret == null) ? tmp : ret;
dynamicDescriptors.remove(name);
return ret;
}
}
protected void putCoreToOrigName(SolrCore c, String name) {
synchronized (locker) {
coreToOrigName.put(c, name);
}
}
protected void removeCoreToOrigName(SolrCore newCore, SolrCore core) {
synchronized (locker) {
String origName = coreToOrigName.remove(core);
if (origName != null) {
coreToOrigName.put(newCore, origName);
}
}
}
protected SolrCore getCoreFromAnyList(String name) {
SolrCore core;
synchronized (locker) {
core = cores.get(name);
if (core != null) {
return core;
}
if (dynamicDescriptors.size() == 0) {
return null; // Nobody even tried to define any transient cores, so we're done.
}
// Now look for already loaded transient cores.
return transientCores.get(name);
}
}
protected CoreDescriptor getDynamicDescriptor(String name) {
synchronized (locker) {
return dynamicDescriptors.get(name);
}
}
protected boolean isLoaded(String name) {
synchronized (locker) {
if (cores.containsKey(name)) {
return true;
}
if (transientCores.containsKey(name)) {
return true;
}
}
return false;
}
protected CoreDescriptor getUnloadedCoreDescriptor(String cname) {
synchronized (locker) {
CoreDescriptor desc = dynamicDescriptors.get(cname);
if (desc == null) {
return null;
}
return new CoreDescriptor(desc);
}
}
protected String getCoreToOrigName(SolrCore solrCore) {
synchronized (locker) {
return coreToOrigName.get(solrCore);
}
}
protected void publishCoresAsDown(ZkController zkController) {
synchronized (locker) {
for (SolrCore core : cores.values()) {
try {
zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
} catch (KeeperException e) {
CoreContainer.log.error("", e);
} catch (InterruptedException e) {
CoreContainer.log.error("", e);
}
}
for (SolrCore core : transientCores.values()) {
try {
zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
} catch (KeeperException e) {
CoreContainer.log.error("", e);
} catch (InterruptedException e) {
CoreContainer.log.error("", e);
}
}
}
}
// Irrepressably ugly bit of the transition in SOLR-4196, but there as at least one test case that follows
// this path, presumably it's there for a reason.
// This is really perverse, but all we need the here is to call a couple of static methods that for back-compat
// purposes
public void persistCores(Config cfg, Properties containerProperties,
Map<String,String> rootSolrAttribs, Map<String,String> coresAttribs,
File file, File configFile, SolrResourceLoader loader) throws XPathExpressionException {
// This is expensive in the maximal case, but I think necessary. It should
// keep a reference open to all of the
// current cores while they are saved. Remember that especially the
// transient core can come and go.
//
// TODO: 5.0. remove the possibility of storing core descriptors in
// solr.xml?
//
List<SolrXMLSerializer.SolrCoreXMLDef> solrCoreXMLDefs = new ArrayList<SolrXMLSerializer.SolrCoreXMLDef>();
synchronized (locker) {
persistCores(cfg, cores, loader, solrCoreXMLDefs);
persistCores(cfg, transientCores, loader, solrCoreXMLDefs);
// add back all the cores that aren't loaded, either in cores or transient
// cores
for (Map.Entry<String,CoreDescriptor> ent : dynamicDescriptors.entrySet()) {
if (!cores.containsKey(ent.getKey())
&& !transientCores.containsKey(ent.getKey())) {
addPersistOneCore(cfg, loader, ent.getValue(), null, solrCoreXMLDefs);
}
}
for (Map.Entry<String,SolrCore> ent : createdCores.entrySet()) {
if (!cores.containsKey(ent.getKey())
&& !transientCores.containsKey(ent.getKey())
&& !dynamicDescriptors.containsKey(ent.getKey())) {
addPersistOneCore(cfg, loader, ent.getValue().getCoreDescriptor(),
null, solrCoreXMLDefs);
}
}
SolrXMLSerializer.SolrXMLDef solrXMLDef = new SolrXMLSerializer.SolrXMLDef();
solrXMLDef.coresDefs = solrCoreXMLDefs;
solrXMLDef.containerProperties = containerProperties;
solrXMLDef.solrAttribs = rootSolrAttribs;
solrXMLDef.coresAttribs = coresAttribs;
SOLR_XML_SERIALIZER.persistFile(file, solrXMLDef);
}
}
// Wait here until any pending operations (load, unload or reload) are completed on this core.
protected SolrCore waitAddPendingCoreOps(String name) {
// Keep multiple threads from operating on a core at one time.
synchronized (locker) {
boolean pending;
do { // Are we currently doing anything to this core? Loading, unloading, reloading?
pending = pendingCoreOps.contains(name); // wait for the core to be done being operated upon
if (! pending) { // Linear list, but shouldn't be too long
for (SolrCore core : pendingCloses) {
if (core.getName().equals(name)) {
pending = true;
break;
}
}
}
if (container.isShutDown()) return null; // Just stop already.
if (pending) {
try {
locker.wait();
} catch (InterruptedException e) {
return null; // Seems best not to do anything at all if the thread is interrupted
}
}
} while (pending);
// We _really_ need to do this within the synchronized block!
if (! container.isShutDown()) {
if (! pendingCoreOps.add(name)) {
CoreContainer.log.warn("Replaced an entry in pendingCoreOps {}, we should not be doing this", name);
}
return getCoreFromAnyList(name); // we might have been _unloading_ the core, so return the core if it was loaded.
}
}
return null;
}
// We should always be removing the first thing in the list with our name! The idea here is to NOT do anything n
// any core while some other operation is working on that core.
protected void removeFromPendingOps(String name) {
synchronized (locker) {
if (! pendingCoreOps.remove(name)) {
CoreContainer.log.warn("Tried to remove core {} from pendingCoreOps and it wasn't there. ", name);
}
locker.notifyAll();
}
}
protected void persistCores(Config cfg, Map<String, SolrCore> whichCores, SolrResourceLoader loader, List<SolrCoreXMLDef> solrCoreXMLDefs) throws XPathExpressionException {
for (SolrCore solrCore : whichCores.values()) {
addPersistOneCore(cfg, loader, solrCore.getCoreDescriptor(), getCoreToOrigName(solrCore), solrCoreXMLDefs);
}
}
private void addCoreProperty(Map<String,String> coreAttribs, SolrResourceLoader loader, Node node, String name,
String value, String defaultValue) {
if (node == null) {
coreAttribs.put(name, value);
return;
}
if (node != null) {
String rawAttribValue = DOMUtil.getAttr(node, name, null);
if (value == null) {
coreAttribs.put(name, rawAttribValue);
return;
}
if (rawAttribValue == null && defaultValue != null && value.equals(defaultValue)) {
return;
}
if (rawAttribValue != null && value.equals(DOMUtil.substituteProperty(rawAttribValue, loader.getCoreProperties()))){
coreAttribs.put(name, rawAttribValue);
} else {
coreAttribs.put(name, value);
}
}
}
protected void addPersistOneCore(Config cfg, SolrResourceLoader loader,
CoreDescriptor dcore, String origCoreName,
List<SolrCoreXMLDef> solrCoreXMLDefs) throws XPathExpressionException {
String coreName = dcore.getProperty(CoreDescriptor.CORE_NAME);
Map<String,String> coreAttribs = new HashMap<String,String>();
CloudDescriptor cd = dcore.getCloudDescriptor();
String collection = null;
if (cd != null) collection = cd.getCollectionName();
if (origCoreName == null) {
origCoreName = coreName;
}
Properties properties = dcore.getCoreProperties();
Node node = null;
if (cfg != null) {
node = cfg.getNode("/solr/cores/core[@name='" + origCoreName + "']",
false);
}
coreAttribs.put(CoreDescriptor.CORE_NAME, coreName);
addCoreProperty(coreAttribs, loader, node, CoreDescriptor.CORE_INSTDIR, dcore.getRawInstanceDir(), null);
coreAttribs.put(CoreDescriptor.CORE_COLLECTION,
StringUtils.isNotBlank(collection) ? collection : dcore.getName());
addCoreProperty(coreAttribs, loader, node, CoreDescriptor.CORE_DATADIR, dcore.getDataDir(), null);
addCoreProperty(coreAttribs, loader, node, CoreDescriptor.CORE_ULOGDIR, dcore.getUlogDir(), null);
addCoreProperty(coreAttribs, loader, node, CoreDescriptor.CORE_TRANSIENT, Boolean.toString(dcore.isTransient()), null);
addCoreProperty(coreAttribs, loader, node, CoreDescriptor.CORE_LOADONSTARTUP, Boolean.toString(dcore.isLoadOnStartup()), null);
addCoreProperty(coreAttribs, loader, node, CoreDescriptor.CORE_COLLECTION,
collection, dcore.getName());
String shard = null;
String roles = null;
if (cd != null) {
shard = cd.getShardId();
roles = cd.getRoles();
}
addCoreProperty(coreAttribs, loader, node, CoreDescriptor.CORE_SHARD,
shard, null);
addCoreProperty(coreAttribs, loader, node, CoreDescriptor.CORE_ROLES,
roles, null);
coreAttribs.put(CoreDescriptor.CORE_LOADONSTARTUP,
Boolean.toString(dcore.isLoadOnStartup()));
coreAttribs.put(CoreDescriptor.CORE_TRANSIENT,
Boolean.toString(dcore.isTransient()));
SolrXMLSerializer.SolrCoreXMLDef solrCoreXMLDef = new SolrXMLSerializer.SolrCoreXMLDef();
solrCoreXMLDef.coreAttribs = coreAttribs;
solrCoreXMLDef.coreProperties = properties;
solrCoreXMLDefs.add(solrCoreXMLDef);
}
protected Object getLocker() { return locker; }
// Be a little careful. We don't want to either open or close a core unless it's _not_ being opened or closed by
// another thread. So within this lock we'll walk along the list of pending closes until we find something NOT in
// the list of threads currently being loaded or reloaded. The "usual" case will probably return the very first
// one anyway..
protected SolrCore getCoreToClose() {
synchronized (locker) {
for (SolrCore core : pendingCloses) {
if (! pendingCoreOps.contains(core.getName())) {
pendingCoreOps.add(core.getName());
pendingCloses.remove(core);
return core;
}
}
}
return null;
}
protected void addCreated(SolrCore core) {
synchronized (locker) {
createdCores.put(core.getName(), core);
}
}
protected String checkUniqueDataDir(String targetPath) {
// Have to check
// loaded cores
// transient cores
// dynamic cores
synchronized (locker) {
for (SolrCore core : cores.values()) {
if (targetPath.equals(core.getDataDir())) return core.getName();
}
for (SolrCore core : transientCores.values()) {
if (targetPath.equals(core.getDataDir())) return core.getName();
}
for (CoreDescriptor desc : dynamicDescriptors.values()) {
if (targetPath.equals(desc.getDataDir())) return desc.getName();
}
}
return null;
}
}

97
solr/core/src/java/org/apache/solr/util/DOMUtil.java
View File

@ -301,6 +301,103 @@ public class DOMUtil {
}
}
}
public static String substituteProperty(String value, Properties coreProperties) {
if (value == null || value.indexOf('$') == -1) {
return value;
}
List<String> fragments = new ArrayList<String>();
List<String> propertyRefs = new ArrayList<String>();
parsePropertyString(value, fragments, propertyRefs);
StringBuilder sb = new StringBuilder();
Iterator<String> i = fragments.iterator();
Iterator<String> j = propertyRefs.iterator();
while (i.hasNext()) {
String fragment = i.next();
if (fragment == null) {
String propertyName = j.next();
String defaultValue = null;
int colon_index = propertyName.indexOf(':');
if (colon_index > -1) {
defaultValue = propertyName.substring(colon_index + 1);
propertyName = propertyName.substring(0,colon_index);
}
if (coreProperties != null) {
fragment = coreProperties.getProperty(propertyName);
}
if (fragment == null) {
fragment = System.getProperty(propertyName, defaultValue);
}
if (fragment == null) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "No system property or default value specified for " + propertyName + " value:" + value);
}
}
sb.append(fragment);
}
return sb.toString();
}
/*
* This method borrowed from Ant's PropertyHelper.parsePropertyStringDefault:
* http://svn.apache.org/repos/asf/ant/core/trunk/src/main/org/apache/tools/ant/PropertyHelper.java
*/
private static void parsePropertyString(String value, List<String> fragments, List<String> propertyRefs) {
int prev = 0;
int pos;
//search for the next instance of $ from the 'prev' position
while ((pos = value.indexOf("$", prev)) >= 0) {
//if there was any text before this, add it as a fragment
//TODO, this check could be modified to go if pos>prev;
//seems like this current version could stick empty strings
//into the list
if (pos > 0) {
fragments.add(value.substring(prev, pos));
}
//if we are at the end of the string, we tack on a $
//then move past it
if (pos == (value.length() - 1)) {
fragments.add("$");
prev = pos + 1;
} else if (value.charAt(pos + 1) != '{') {
//peek ahead to see if the next char is a property or not
//not a property: insert the char as a literal
/*
fragments.addElement(value.substring(pos + 1, pos + 2));
prev = pos + 2;
*/
if (value.charAt(pos + 1) == '$') {
//backwards compatibility two $ map to one mode
fragments.add("$");
prev = pos + 2;
} else {
//new behaviour: $X maps to $X for all values of X!='$'
fragments.add(value.substring(pos, pos + 2));
prev = pos + 2;
}
} else {
//property found, extract its name or bail on a typo
int endName = value.indexOf('}', pos);
if (endName < 0) {
throw new RuntimeException("Syntax error in property: " + value);
}
String propertyName = value.substring(pos + 2, endName);
fragments.add(null);
propertyRefs.add(propertyName);
prev = endName + 1;
}
}
//no more $ signs found
//if there is any tail to the string, append it
if (prev < value.length()) {
fragments.add(value.substring(prev));
}
}
}

20
solr/core/src/test/org/apache/solr/core/TestSolrXml.java
View File

@ -20,9 +20,11 @@ package org.apache.solr.core;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.Test;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -38,7 +40,14 @@ public class TestSolrXml extends SolrTestCaseJ4 {
FileUtils.copyFile(new File(testSrcRoot, "solr-50-all.xml"), new File(solrHome, "solr.xml"));
try {
InputStream is = new FileInputStream(new File(solrHome, "solr.xml"));
ConfigSolr cfg = new ConfigSolrXml(new SolrResourceLoader("solr/collection1"), null, is, null, false, null);
Config config = new Config(new SolrResourceLoader("solr/collection1"), null, new InputSource(is), null, false);
boolean oldStyle = (config.getNode("solr/cores", false) != null);
ConfigSolr cfg;
if (oldStyle) {
cfg = new ConfigSolrXmlOld(config, cc);
} else {
cfg = new ConfigSolrXml(config, cc);
}
assertEquals("Did not find expected value", cfg.get(ConfigSolr.CfgProp.SOLR_ADMINHANDLER, null), "testAdminHandler");
assertEquals("Did not find expected value", cfg.getInt(ConfigSolr.CfgProp.SOLR_CORELOADTHREADS, 0), 11);
@ -90,7 +99,14 @@ public class TestSolrXml extends SolrTestCaseJ4 {
FileUtils.copyFile(new File(testSrcRoot, "solr-50-all.xml"), new File(solrHome, "solr.xml"));
try {
InputStream is = new FileInputStream(new File(solrHome, "solr.xml"));
ConfigSolr cfg = new ConfigSolrXml(new SolrResourceLoader("solr/collection1"), null, is, null, false, null);
Config config = new Config(new SolrResourceLoader("solr/collection1"), null, new InputSource(is), null, false);
boolean oldStyle = (config.getNode("solr/cores", false) != null);
ConfigSolr cfg;
if (oldStyle) {
cfg = new ConfigSolrXmlOld(config, cc);
} else {
cfg = new ConfigSolrXml(config, cc);
}
assertEquals("Did not find expected value", cfg.get(ConfigSolr.CfgProp.SOLR_COREROOTDIRECTORY, null), "myCoreRoot");
assertEquals("Did not find expected value", cfg.getInt(ConfigSolr.CfgProp.SOLR_HOSTPORT, 0), 8888);
assertEquals("Did not find expected value", cfg.getInt(ConfigSolr.CfgProp.SOLR_SHARDHANDLERFACTORY_CONNTIMEOUT, 0), 200);

37
solr/test-framework/src/java/org/apache/solr/util/TestHarness.java
View File

@ -20,6 +20,8 @@ package org.apache.solr.util;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.Config;
import org.apache.solr.core.ConfigSolrXmlOld;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.CoreContainer;
@ -39,13 +41,18 @@ import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.servlet.DirectSolrConnection;
import org.apache.solr.common.util.NamedList.NamedListEntry;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.util.HashMap;
import java.util.Map;
import javax.xml.parsers.ParserConfigurationException;
/**
* This class provides a simple harness that may be useful when
@ -171,15 +178,27 @@ public class TestHarness extends BaseTestHarness {
}
@Override
public CoreContainer initialize() {
CoreContainer container = new CoreContainer(new SolrResourceLoader(SolrResourceLoader.locateSolrHome())) {
{
hostPort = System.getProperty("hostPort");
hostContext = "solr";
defaultCoreName = CoreContainer.DEFAULT_DEFAULT_CORE_NAME;
initShardHandler();
initZooKeeper(System.getProperty("zkHost"), 10000);
}
};
CoreContainer container;
try {
container = new CoreContainer(new SolrResourceLoader(SolrResourceLoader.locateSolrHome())) {
{
hostPort = System.getProperty("hostPort");
hostContext = "solr";
defaultCoreName = CoreContainer.DEFAULT_DEFAULT_CORE_NAME;
initShardHandler();
initZooKeeper(System.getProperty("zkHost"), 10000);
ByteArrayInputStream is = new ByteArrayInputStream(ConfigSolrXmlOld.DEF_SOLR_XML.getBytes("UTF-8"));
Config config = new Config(loader, null, new InputSource(is), null, false);
cfg = new ConfigSolrXmlOld(config, this);
}
};
} catch (ParserConfigurationException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
} catch (SAXException e) {
throw new RuntimeException(e);
}
LogWatcher<?> logging = new JulWatcher("test");
logging.registerListener(new ListenerConfig(), container);
container.setLogging(logging);