Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4749: Clean up and refactor CoreContainer code around solr.xml and SolrCore management. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1470758 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-04-23 00:09:30 +00:00
parent 7410b25972
commit 33abacf125
9 changed files with 528 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/core/src/java/org/apache/solr/core/ConfigSolr.java
View File

@ -38,7 +38,6 @@ import org.w3c.dom.NodeList;
public abstract class ConfigSolr {
protected static Logger log = LoggerFactory.getLogger(ConfigSolr.class);
public final static String CORE_PROP_FILE = "core.properties";
public final static String SOLR_XML_FILE = "solr.xml";
// Ugly for now, but we'll at least be able to centralize all of the differences between 4x and 5x.
@ -132,8 +131,6 @@ public abstract class ConfigSolr {
public abstract void substituteProperties();
public abstract String getCoreNameFromOrig(String origCoreName, SolrResourceLoader loader, String coreName);
public abstract List<String> getAllCoreNames();
public abstract String getProperty(String coreName, String property, String defaultVal);

138
solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
View File

@ -17,7 +17,6 @@ package org.apache.solr.core;
* limitations under the License.
*/
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -31,15 +30,10 @@ import java.util.Properties;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.io.IOUtils;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.PropertiesUtil;
import org.apache.solr.util.SystemIdResolver;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
@ -49,15 +43,21 @@ import org.xml.sax.SAXException;
public class ConfigSolrXml extends ConfigSolr {
protected static Logger log = LoggerFactory.getLogger(ConfigSolrXml.class);
private final Map<String, CoreDescriptor> coreDescriptorMap = new HashMap<String, CoreDescriptor>();
private SolrCoreDiscoverer solrCoreDiscoverer = new SolrCoreDiscoverer();
private final Map<String, CoreDescriptor> coreDescriptorMap;
public ConfigSolrXml(Config config, CoreContainer container)
throws ParserConfigurationException, IOException, SAXException {
super(config);
init(container);
checkForIllegalConfig(container);
fillPropMap();
String coreRoot = get(CfgProp.SOLR_COREROOTDIRECTORY, container.getSolrHome());
coreDescriptorMap = solrCoreDiscoverer.discover(container, new File(coreRoot));
}
private void init(CoreContainer container) throws IOException {
private void checkForIllegalConfig(CoreContainer container) throws IOException {
// Do sanity checks - we don't want to find old style config
failIfFound("solr/@coreLoadThreads");
@ -88,9 +88,7 @@ public class ConfigSolrXml extends ConfigSolr {
failIfFound("solr/cores/@defaultCoreName");
failIfFound("solr/@persistent");
failIfFound("solr/cores/@adminPath");
fillPropMap();
initCoreList(container);
}
private void failIfFound(String xPath) {
@ -137,17 +135,7 @@ public class ConfigSolrXml extends ConfigSolr {
propMap.put(CfgProp.SOLR_SHARDHANDLERFACTORY_SOCKETTIMEOUT, doSub("solr/shardHandlerFactory/int[@name='socketTimeout']"));
}
private void initCoreList(CoreContainer container) throws IOException {
if (container != null) { // TODO: 5.0. Yet another bit of nonsense only
// because of the test harness.
String coreRoot = get(CfgProp.SOLR_COREROOTDIRECTORY,
container.getSolrHome());
walkFromHere(new File(coreRoot), container, new HashMap<String,String>(),
new HashMap<String,String>());
}
}
@Override
public Map<String,String> readCoreAttributes(String coreName) {
@ -156,101 +144,6 @@ public class ConfigSolrXml extends ConfigSolr {
return attrs; // this is a no-op.... intentionally
}
// Basic recursive tree walking, looking for "core.properties" files. Once one is found, we'll stop going any
// deeper in the tree.
//
// @param file - the directory we're to either read the properties file from or recurse into.
private void walkFromHere(File file, CoreContainer container, Map<String, String> seenDirs, HashMap<String, String> seenCores)
throws IOException {
log.info("Looking for cores in " + file.getCanonicalPath());
if (! file.exists()) return;
for (File childFile : file.listFiles()) {
// This is a little tricky, we are asking if core.properties exists in a child directory of the directory passed
// in. In other words we're looking for core.properties in the grandchild directories of the parameter passed
// in. That allows us to gracefully top recursing deep but continue looking wide.
File propFile = new File(childFile, CORE_PROP_FILE);
if (propFile.exists()) { // Stop looking after processing this file!
addCore(container, seenDirs, seenCores, childFile, propFile);
continue; // Go on to the sibling directory, don't descend any deeper.
}
if (childFile.isDirectory()) {
walkFromHere(childFile, container, seenDirs, seenCores);
}
}
}
private void addCore(CoreContainer container, Map<String, String> seenDirs, Map<String, String> seenCores,
File childFile, File propFile) throws IOException {
log.info("Discovered properties file {}, adding to cores", propFile.getAbsolutePath());
Properties propsOrig = new Properties();
InputStream is = new FileInputStream(propFile);
try {
propsOrig.load(is);
} finally {
IOUtils.closeQuietly(is);
}
Properties props = new Properties();
for (String prop : propsOrig.stringPropertyNames()) {
props.put(prop, PropertiesUtil.substituteProperty(propsOrig.getProperty(prop), null));
}
// Too much of the code depends on this value being here, but it is NOT supported in discovery mode, so
// ignore it if present in the core.properties file.
props.setProperty(CoreDescriptor.CORE_INSTDIR, childFile.getPath());
if (props.getProperty(CoreDescriptor.CORE_NAME) == null) {
// Should default to this directory
props.setProperty(CoreDescriptor.CORE_NAME, childFile.getName());
}
CoreDescriptor desc = new CoreDescriptor(container, props);
coreDescriptorMap.put(desc.getName(), desc);
}
public IndexSchema getSchemaFromZk(ZkController zkController, String zkConfigName, String schemaName,
SolrConfig config)
throws KeeperException, InterruptedException {
byte[] configBytes = zkController.getConfigFileData(zkConfigName, schemaName);
InputSource is = new InputSource(new ByteArrayInputStream(configBytes));
is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(schemaName));
IndexSchema schema = new IndexSchema(config, schemaName, is);
return schema;
}
@Override
public String getCoreNameFromOrig(String origCoreName,
SolrResourceLoader loader, String coreName) {
// first look for an exact match
for (Map.Entry<String,CoreDescriptor> ent : coreDescriptorMap
.entrySet()) {
String name = ent.getValue().getProperty(CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(name)) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
for (Map.Entry<String,CoreDescriptor> ent : coreDescriptorMap
.entrySet()) {
String name = ent.getValue().getProperty(CoreDescriptor.CORE_NAME, null);
// see if we match with substitution
if (origCoreName.equals(PropertiesUtil.substituteProperty(name,
loader.getCoreProperties()))) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
return null;
}
@Override
public List<String> getAllCoreNames() {
List<String> ret = new ArrayList<String>(coreDescriptorMap.keySet());
@ -260,7 +153,6 @@ public class ConfigSolrXml extends ConfigSolr {
@Override
public String getProperty(String coreName, String property, String defaultVal) {
CoreDescriptor cd = coreDescriptorMap.get(coreName);
if (cd == null) return defaultVal;
@ -269,11 +161,9 @@ public class ConfigSolrXml extends ConfigSolr {
@Override
public Properties readCoreProperties(String coreName) {
CoreDescriptor plus = coreDescriptorMap.get(coreName);
if (plus == null) return null;
return new Properties(plus.getCoreProperties());
CoreDescriptor cd = coreDescriptorMap.get(coreName);
if (cd == null) return null;
return new Properties(cd.getCoreProperties());
}
static Properties getCoreProperties(String instanceDir, CoreDescriptor dcore) {

53
solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
View File

@ -37,7 +37,6 @@ import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.PropertiesUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.NamedNodeMap;
@ -58,11 +57,13 @@ public class ConfigSolrXmlOld extends ConfigSolr {
throws ParserConfigurationException, IOException, SAXException {
super(config);
init(container);
checkForIllegalConfig(container);
fillPropMap();
initCoreList(container);
}
private void init(CoreContainer container) throws IOException {
private void checkForIllegalConfig(CoreContainer container) throws IOException {
// Do sanity checks - we don't want to find new style
// config
failIfFound("solr/str[@name='adminHandler']");
@ -86,9 +87,6 @@ public class ConfigSolrXmlOld extends ConfigSolr {
failIfFound("solr/logging/watcher/int[@name='size']");
failIfFound("solr/logging/watcher/int[@name='threshold']");
fillPropMap();
initCoreList(container);
}
private void failIfFound(String xPath) {
@ -224,47 +222,6 @@ public class ConfigSolrXmlOld extends ConfigSolr {
return attrs;
}
// Fortunately, we don't iterate over these too often, so the waste is probably tolerable.
@Override
public String getCoreNameFromOrig(String origCoreName,
SolrResourceLoader loader, String coreName) {
// look for an existing node
synchronized (coreNodes) {
// first look for an exact match
Node coreNode = null;
for (int i = 0; i < coreNodes.getLength(); i++) {
Node node = coreNodes.item(i);
String name = DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(name)) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
if (coreNode == null) {
// see if we match with substitution
for (int i = 0; i < coreNodes.getLength(); i++) {
Node node = coreNodes.item(i);
String name = DOMUtil.getAttr(node, CoreDescriptor.CORE_NAME, null);
if (origCoreName.equals(PropertiesUtil.substituteProperty(name,
loader.getCoreProperties()))) {
if (coreName.equals(origCoreName)) {
return name;
}
return coreName;
}
}
}
}
return null;
}
@Override
public List<String> getAllCoreNames() {
List<String> ret = new ArrayList<String>();

355
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -21,10 +21,8 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@ -45,7 +43,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
@ -55,13 +52,10 @@ import javax.xml.transform.dom.DOMSource;
import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.io.IOUtils;
import org.apache.solr.cloud.CurrentCoreDescriptorProvider;
import org.apache.solr.cloud.SolrZkServer;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.handler.admin.CollectionsHandler;
@ -74,11 +68,9 @@ import org.apache.solr.logging.jul.JulWatcher;
import org.apache.solr.logging.log4j.Log4jWatcher;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.update.SolrCoreState;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.FileUtils;
import org.apache.solr.util.PropertiesUtil;
import org.apache.solr.util.SystemIdResolver;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@ -97,10 +89,7 @@ public class CoreContainer
{
private static final String LEADER_VOTE_WAIT = "180000"; // 3 minutes
private static final int CORE_LOAD_THREADS = 3;
/** @deprecated will be remove in Solr 5.0 (SOLR-4622) */
private static final String DEFAULT_HOST_CONTEXT = "solr";
/** @deprecated will be remove in Solr 5.0 (SOLR-4622) */
private static final String DEFAULT_HOST_PORT = "8983";
private static final int DEFAULT_ZK_CLIENT_TIMEOUT = 15000;
public static final String DEFAULT_DEFAULT_CORE_NAME = "collection1";
private static final boolean DEFAULT_SHARE_SCHEMA = false;
@ -116,9 +105,7 @@ public class CoreContainer
protected boolean persistent = false;
protected String adminPath = null;
protected volatile String managementPath = null;
protected String hostPort;
protected String hostContext;
protected String host;
protected CoreAdminHandler coreAdminHandler = null;
protected CollectionsHandler collectionsHandler = null;
protected File configFile = null;
@ -132,16 +119,13 @@ public class CoreContainer
protected String solrHome;
protected String defaultCoreName = null;
private ZkController zkController;
private SolrZkServer zkServer;
protected ZkContainer zkSys = new ZkContainer();
private ShardHandlerFactory shardHandlerFactory;
protected LogWatcher logging = null;
private String zkHost;
private int transientCacheSize = Integer.MAX_VALUE;
private String leaderVoteWait = LEADER_VOTE_WAIT;
private int distribUpdateConnTimeout = 0;
private int distribUpdateSoTimeout = 0;
private int coreLoadThreads;
private CloserThread backgroundCloser = null;
protected volatile ConfigSolr cfg;
@ -182,153 +166,6 @@ public class CoreContainer
this.solrHome = solrHome;
}
public SolrConfig getSolrConfigFromZk(String zkConfigName, String solrConfigFileName,
SolrResourceLoader resourceLoader) {
SolrConfig cfg = null;
try {
byte[] config = zkController.getConfigFileData(zkConfigName,
solrConfigFileName);
InputSource is = new InputSource(new ByteArrayInputStream(config));
is.setSystemId(SystemIdResolver
.createSystemIdFromResourceName(solrConfigFileName));
cfg = solrConfigFileName == null ? new SolrConfig(resourceLoader,
SolrConfig.DEFAULT_CONF_FILE, is) : new SolrConfig(resourceLoader,
solrConfigFileName, is);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"getSolrConfigFromZK failed for " + zkConfigName + " "
+ solrConfigFileName, e);
}
return cfg;
}
protected void initZooKeeper(String zkHost, int zkClientTimeout) {
// if zkHost sys property is not set, we are not using ZooKeeper
String zookeeperHost;
if(zkHost == null) {
zookeeperHost = System.getProperty("zkHost");
} else {
zookeeperHost = zkHost;
}
String zkRun = System.getProperty("zkRun");
if (zkRun == null && zookeeperHost == null)
return; // not in zk mode
// BEGIN: SOLR-4622: deprecated hardcoded defaults for hostPort & hostContext
if (null == hostPort) {
// throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
// "'hostPort' must be configured to run SolrCloud");
log.warn("Solr 'hostPort' has not be explicitly configured, using hardcoded default of " + DEFAULT_HOST_PORT + ". This default has been deprecated and will be removed in future versions of Solr, please configure this value explicitly");
hostPort = DEFAULT_HOST_PORT;
}
if (null == hostContext) {
// throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
// "'hostContext' must be configured to run SolrCloud");
log.warn("Solr 'hostContext' has not be explicitly configured, using hardcoded default of " + DEFAULT_HOST_CONTEXT + ". This default has been deprecated and will be removed in future versions of Solr, please configure this value explicitly");
hostContext = DEFAULT_HOST_CONTEXT;
}
// END: SOLR-4622
// zookeeper in quorum mode currently causes a failure when trying to
// register log4j mbeans. See SOLR-2369
// TODO: remove after updating to an slf4j based zookeeper
System.setProperty("zookeeper.jmx.log4j.disable", "true");
if (zkRun != null) {
String zkDataHome = System.getProperty("zkServerDataDir", solrHome + "zoo_data");
String zkConfHome = System.getProperty("zkServerConfDir", solrHome);
zkServer = new SolrZkServer(zkRun, zookeeperHost, zkDataHome, zkConfHome, hostPort);
zkServer.parseConfig();
zkServer.start();
// set client from server config if not already set
if (zookeeperHost == null) {
zookeeperHost = zkServer.getClientString();
}
}
int zkClientConnectTimeout = 15000;
if (zookeeperHost != null) {
// we are ZooKeeper enabled
try {
// If this is an ensemble, allow for a long connect time for other servers to come up
if (zkRun != null && zkServer.getServers().size() > 1) {
zkClientConnectTimeout = 24 * 60 * 60 * 1000; // 1 day for embedded ensemble
log.info("Zookeeper client=" + zookeeperHost + " Waiting for a quorum.");
} else {
log.info("Zookeeper client=" + zookeeperHost);
}
String confDir = System.getProperty("bootstrap_confdir");
boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
if(!ZkController.checkChrootPath(zookeeperHost, (confDir!=null) || boostrapConf)) {
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"A chroot was specified in ZkHost but the znode doesn't exist. ");
}
zkController = new ZkController(this, zookeeperHost, zkClientTimeout,
zkClientConnectTimeout, host, hostPort, hostContext,
leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout,
new CurrentCoreDescriptorProvider() {
@Override
public List<CoreDescriptor> getCurrentDescriptors() {
List<CoreDescriptor> descriptors = new ArrayList<CoreDescriptor>(
getCoreNames().size());
for (SolrCore core : getCores()) {
descriptors.add(core.getCoreDescriptor());
}
return descriptors;
}
});
if (zkRun != null && zkServer.getServers().size() > 1 && confDir == null && boostrapConf == false) {
// we are part of an ensemble and we are not uploading the config - pause to give the config time
// to get up
Thread.sleep(10000);
}
if(confDir != null) {
File dir = new File(confDir);
if(!dir.isDirectory()) {
throw new IllegalArgumentException("bootstrap_confdir must be a directory of configuration files");
}
String confName = System.getProperty(ZkController.COLLECTION_PARAM_PREFIX+ZkController.CONFIGNAME_PROP, "configuration1");
zkController.uploadConfigDir(dir, confName);
}
if(boostrapConf) {
ZkController.bootstrapConf(zkController.getZkClient(), cfg, solrHome);
}
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (TimeoutException e) {
log.error("Could not connect to ZooKeeper", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (IOException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
}
}
}
public Properties getContainerProperties() {
return containerProperties;
}
@ -405,18 +242,22 @@ public class CoreContainer
this.loader = new SolrResourceLoader(dir);
solrHome = loader.getInstanceDir();
//ConfigSolr cfg;
// keep orig config for persist to consult. TODO: Remove this silly stuff for 5.0, persistence not supported.
try {
Config config = new Config(loader, null, new InputSource(is), null, false);
this.origCfg = new Config(loader, null, copyDoc(config.getDocument()));
// old style defines cores in solr.xml, new style disovers them by
// directory structure
boolean oldStyle = (config.getNode("solr/cores", false) != null);
if (oldStyle) {
// ConfigSolr handles keep orig values around for non solrcore level items,
// but this is still how original core lvl attributes are kept around
this.origCfg = new Config(loader, null, copyDoc(config.getDocument()));
this.cfg = new ConfigSolrXmlOld(config, this);
} else {
this.cfg = new ConfigSolrXml(config, this);
}
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "", e);
@ -501,17 +342,17 @@ public class CoreContainer
shareSchema = cfg.getBool(ConfigSolr.CfgProp.SOLR_SHARESCHEMA, DEFAULT_SHARE_SCHEMA);
zkClientTimeout = cfg.getInt(ConfigSolr.CfgProp.SOLR_ZKCLIENTTIMEOUT, DEFAULT_ZK_CLIENT_TIMEOUT);
distribUpdateConnTimeout = cfg.getInt(ConfigSolr.CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, 0);
distribUpdateSoTimeout = cfg.getInt(ConfigSolr.CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, 0);
int distribUpdateConnTimeout = cfg.getInt(ConfigSolr.CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, 0);
int distribUpdateSoTimeout = cfg.getInt(ConfigSolr.CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, 0);
// Note: initZooKeeper will apply hardcoded default if cloud mode
hostPort = cfg.get(ConfigSolr.CfgProp.SOLR_HOSTPORT, null);
String hostPort = cfg.get(ConfigSolr.CfgProp.SOLR_HOSTPORT, null);
// Note: initZooKeeper will apply hardcoded default if cloud mode
hostContext = cfg.get(ConfigSolr.CfgProp.SOLR_HOSTCONTEXT, null);
String hostContext = cfg.get(ConfigSolr.CfgProp.SOLR_HOSTCONTEXT, null);
host = cfg.get(ConfigSolr.CfgProp.SOLR_HOST, null);
String host = cfg.get(ConfigSolr.CfgProp.SOLR_HOST, null);
leaderVoteWait = cfg.get(ConfigSolr.CfgProp.SOLR_LEADERVOTEWAIT, LEADER_VOTE_WAIT);
String leaderVoteWait = cfg.get(ConfigSolr.CfgProp.SOLR_LEADERVOTEWAIT, LEADER_VOTE_WAIT);
adminHandler = cfg.get(ConfigSolr.CfgProp.SOLR_ADMINHANDLER, null);
managementPath = cfg.get(ConfigSolr.CfgProp.SOLR_MANAGEMENTPATH, null);
@ -524,7 +365,7 @@ public class CoreContainer
zkClientTimeout = Integer.parseInt(System.getProperty("zkClientTimeout",
Integer.toString(zkClientTimeout)));
initZooKeeper(zkHost, zkClientTimeout);
zkSys.initZooKeeper(this, solrHome, zkHost, zkClientTimeout, hostPort, hostContext, host, leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout);
if (isZooKeeperAware() && coreLoadThreads <= 1) {
throw new SolrException(ErrorCode.SERVER_ERROR,
@ -577,7 +418,7 @@ public class CoreContainer
p.setSchemaName(opt);
}
if (zkController != null) {
if (zkSys.getZkController() != null) {
opt = cfg.getProperty(oneCoreName, CoreDescriptor.CORE_SHARD, null);
if (opt != null && opt.length() > 0) {
p.getCloudDescriptor().setShardId(opt);
@ -734,7 +575,7 @@ public class CoreContainer
if (isZooKeeperAware()) {
try {
zkController.publishAndWaitForDownStates();
zkSys.getZkController().publishAndWaitForDownStates();
} catch (KeeperException e) {
log.error("", e);
} catch (InterruptedException e) {
@ -745,7 +586,7 @@ public class CoreContainer
isShutDown = true;
if (isZooKeeperAware()) {
publishCoresAsDown();
zkSys.publishCoresAsDown(solrCores.getCores());
cancelCoreRecoveries();
}
@ -780,24 +621,21 @@ public class CoreContainer
}
// we want to close zk stuff last
if (zkController != null) {
zkController.close();
}
if (zkServer != null) {
zkServer.stop();
}
zkSys.close();
}
}
public void cancelCoreRecoveries() {
ArrayList<SolrCoreState> coreStates = new ArrayList<SolrCoreState>();
solrCores.addCoresToList(coreStates);
List<SolrCore> cores = solrCores.getCores();
// we must cancel without holding the cores sync
// make sure we wait for any recoveries to stop
for (SolrCoreState coreState : coreStates) {
for (SolrCore core : cores) {
try {
coreState.cancelRecovery();
core.getSolrCoreState().cancelRecovery();
} catch (Throwable t) {
SolrException.log(log, "Error canceling recovery for core", t);
}
@ -825,10 +663,10 @@ public class CoreContainer
throw new RuntimeException( "Invalid core name: "+name );
}
if (zkController != null) {
if (zkSys.getZkController() != null) {
// this happens before we can receive requests
try {
zkController.preRegister(core);
zkSys.getZkController().preRegister(core);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
@ -866,7 +704,7 @@ public class CoreContainer
if( old == null || old == core) {
log.info( "registering core: "+name );
registerInZk(core);
zkSys.registerInZk(core);
return null;
}
else {
@ -874,43 +712,11 @@ public class CoreContainer
if (!returnPrevNotClosed) {
old.close();
}
registerInZk(core);
zkSys.registerInZk(core);
return old;
}
}
private void registerInZk(SolrCore core) {
if (zkController != null) {
try {
zkController.register(core.getName(), core.getCoreDescriptor());
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
SolrException.log(log, "", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "",
e);
} catch (Exception e) {
// if register fails, this is really bad - close the zkController to
// minimize any damage we can cause
try {
zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
} catch (KeeperException e1) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (InterruptedException e1) {
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
}
SolrException.log(log, "", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "",
e);
}
}
}
/**
* Registers a SolrCore descriptor in the registry using the core's name.
* If returnPrev==false, the old core, if different, is closed.
@ -924,43 +730,6 @@ public class CoreContainer
return registerCore(false, name, core, returnPrev);
}
// Helper method to separate out creating a core from ZK as opposed to the "usual" way. See create()
private SolrCore createFromZk(String instanceDir, CoreDescriptor dcore)
{
try {
SolrResourceLoader solrLoader = null;
SolrConfig config = null;
String zkConfigName = null;
IndexSchema schema;
String collection = dcore.getCloudDescriptor().getCollectionName();
zkController.createCollectionZkNode(dcore.getCloudDescriptor());
zkConfigName = zkController.readConfigName(collection);
if (zkConfigName == null) {
log.error("Could not find config name for collection:" + collection);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"Could not find config name for collection:" + collection);
}
solrLoader = new ZkSolrResourceLoader(instanceDir, zkConfigName, loader.getClassLoader(),
ConfigSolrXml.getCoreProperties(instanceDir, dcore), zkController);
config = getSolrConfigFromZk(zkConfigName, dcore.getConfigName(), solrLoader);
schema = IndexSchemaFactory.buildIndexSchema(dcore.getSchemaName(), config);
return new SolrCore(dcore.getName(), null, config, schema, dcore);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
}
}
// Helper method to separate out creating a core from local configuration files. See create()
private SolrCore createFromLocal(String instanceDir, CoreDescriptor dcore) {
SolrResourceLoader solrLoader = null;
@ -1033,8 +802,8 @@ public class CoreContainer
// Initialize the solr config
SolrCore created = null;
if (zkController != null) {
created = createFromZk(instanceDir, dcore);
if (zkSys.getZkController() != null) {
created = zkSys.createFromZk(instanceDir, dcore, loader);
} else {
created = createFromLocal(instanceDir, dcore);
}
@ -1130,21 +899,21 @@ public class CoreContainer
log.info("Reloading SolrCore '{}' using instanceDir: {}",
cd.getName(), instanceDir.getAbsolutePath());
SolrResourceLoader solrLoader;
if(zkController == null) {
if(zkSys.getZkController() == null) {
solrLoader = new SolrResourceLoader(instanceDir.getAbsolutePath(), loader.getClassLoader(), ConfigSolrXml.getCoreProperties(instanceDir.getAbsolutePath(), cd));
} else {
try {
String collection = cd.getCloudDescriptor().getCollectionName();
zkController.createCollectionZkNode(cd.getCloudDescriptor());
zkSys.getZkController().createCollectionZkNode(cd.getCloudDescriptor());
String zkConfigName = zkController.readConfigName(collection);
String zkConfigName = zkSys.getZkController().readConfigName(collection);
if (zkConfigName == null) {
log.error("Could not find config name for collection:" + collection);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"Could not find config name for collection:" + collection);
}
solrLoader = new ZkSolrResourceLoader(instanceDir.getAbsolutePath(), zkConfigName, loader.getClassLoader(),
ConfigSolrXml.getCoreProperties(instanceDir.getAbsolutePath(), cd), zkController);
ConfigSolrXml.getCoreProperties(instanceDir.getAbsolutePath(), cd), zkSys.getZkController());
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
@ -1310,22 +1079,6 @@ public class CoreContainer
return adminPath;
}
public String getHostPort() {
return hostPort;
}
public String getHostContext() {
return hostContext;
}
public String getHost() {
return host;
}
public int getZkClientTimeout() {
return zkClientTimeout;
}
public String getManagementPath() {
return managementPath;
}
@ -1361,6 +1114,7 @@ public class CoreContainer
}
/** Persists the cores config file in cores.xml. */
@Deprecated
public void persist() {
persistFile(configFile);
}
@ -1377,7 +1131,7 @@ public class CoreContainer
}
/** Persists the cores config file in a user provided file. */
//TODO: obsolete in SOLR 5.0
@Deprecated
public void persistFile(File file) {
assert file != null;
// only the old solrxml persists
@ -1398,20 +1152,20 @@ public class CoreContainer
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_SHARESCHEMA,"shareSchema",
Boolean.toString(this.shareSchema),
Boolean.toString(DEFAULT_SHARE_SCHEMA));
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_HOST, "host", this.host, null);
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_HOST, "host", zkSys.getHost(), null);
if (! (null == defaultCoreName || defaultCoreName.equals("")) ) {
coresAttribs.put("defaultCoreName", defaultCoreName);
}
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_HOSTPORT, "hostPort", this.hostPort, DEFAULT_HOST_PORT);
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_HOSTPORT, "hostPort",zkSys.getHostPort(), ZkContainer.DEFAULT_HOST_PORT);
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_ZKCLIENTTIMEOUT, "zkClientTimeout",
intToString(this.zkClientTimeout),
Integer.toString(DEFAULT_ZK_CLIENT_TIMEOUT));
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_HOSTCONTEXT, "hostContext",
this.hostContext, DEFAULT_HOST_CONTEXT);
zkSys.getHostContext(), ZkContainer.DEFAULT_HOST_CONTEXT);
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_LEADERVOTEWAIT, "leaderVoteWait",
this.leaderVoteWait, LEADER_VOTE_WAIT);
zkSys.getLeaderVoteWait(), LEADER_VOTE_WAIT);
addCoresAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_CORELOADTHREADS, "coreLoadThreads",
Integer.toString(this.coreLoadThreads), Integer.toString(CORE_LOAD_THREADS));
if (transientCacheSize != Integer.MAX_VALUE) { // This test
@ -1432,7 +1186,6 @@ public class CoreContainer
return Integer.toString(integer);
}
//TODO: Obsolete in 5.0 Having to pass prop is a hack to get us to 5.0.
private void addCoresAttrib(Map<String,String> coresAttribs, ConfigSolr.CfgProp prop,
String attribName, String attribValue, String defaultValue) {
if (cfg == null) {
@ -1458,11 +1211,11 @@ public class CoreContainer
}
public boolean isZooKeeperAware() {
return zkController != null;
return zkSys.getZkController() != null;
}
public ZkController getZkController() {
return zkController;
return zkSys.getZkController();
}
public boolean isShareSchema() {
@ -1496,20 +1249,6 @@ public class CoreContainer
tx.transform(source,result);
return (Document)result.getNode();
}
protected void publishCoresAsDown() {
List<SolrCore> cores = solrCores.getCores();
for (SolrCore core : cores) {
try {
zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
} catch (KeeperException e) {
CoreContainer.log.error("", e);
} catch (InterruptedException e) {
CoreContainer.log.error("", e);
}
}
}
}
class CloserThread extends Thread {

95
solr/core/src/java/org/apache/solr/core/SolrCoreDiscoverer.java Normal file
View File

@ -0,0 +1,95 @@
package org.apache.solr.core;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.apache.solr.util.PropertiesUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class SolrCoreDiscoverer {
protected static Logger log = LoggerFactory.getLogger(SolrCoreDiscoverer.class);
public final static String CORE_PROP_FILE = "core.properties";
public Map<String, CoreDescriptor> discover(CoreContainer container, File root) throws IOException {
Map<String, CoreDescriptor> coreDescriptorMap = new HashMap<String, CoreDescriptor>();
if (container != null) {
walkFromHere(root, container, coreDescriptorMap);
}
return coreDescriptorMap;
}
// Basic recursive tree walking, looking for "core.properties" files. Once one is found, we'll stop going any
// deeper in the tree.
//
private void walkFromHere(File file, CoreContainer container, Map<String,CoreDescriptor> coreDescriptorMap)
throws IOException {
log.info("Looking for cores in " + file.getCanonicalPath());
if (! file.exists()) return;
for (File childFile : file.listFiles()) {
// This is a little tricky, we are asking if core.properties exists in a child directory of the directory passed
// in. In other words we're looking for core.properties in the grandchild directories of the parameter passed
// in. That allows us to gracefully top recursing deep but continue looking wide.
File propFile = new File(childFile, CORE_PROP_FILE);
if (propFile.exists()) { // Stop looking after processing this file!
addCore(container, childFile, propFile, coreDescriptorMap);
continue; // Go on to the sibling directory, don't descend any deeper.
}
if (childFile.isDirectory()) {
walkFromHere(childFile, container, coreDescriptorMap);
}
}
}
private void addCore(CoreContainer container, File childFile, File propFile, Map<String,CoreDescriptor> coreDescriptorMap) throws IOException {
log.info("Discovered properties file {}, adding to cores", propFile.getAbsolutePath());
Properties propsOrig = new Properties();
InputStream is = new FileInputStream(propFile);
try {
propsOrig.load(is);
} finally {
IOUtils.closeQuietly(is);
}
Properties props = new Properties();
for (String prop : propsOrig.stringPropertyNames()) {
props.put(prop, PropertiesUtil.substituteProperty(propsOrig.getProperty(prop), null));
}
// Too much of the code depends on this value being here, but it is NOT supported in discovery mode, so
// ignore it if present in the core.properties file.
props.setProperty(CoreDescriptor.CORE_INSTDIR, childFile.getPath());
if (props.getProperty(CoreDescriptor.CORE_NAME) == null) {
// Should default to this directory
props.setProperty(CoreDescriptor.CORE_NAME, childFile.getName());
}
CoreDescriptor desc = new CoreDescriptor(container, props);
coreDescriptorMap.put(desc.getName(), desc);
}
}

27
solr/core/src/java/org/apache/solr/core/SolrCores.java
View File

@ -17,13 +17,9 @@ import javax.xml.xpath.XPathExpressionException;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.SolrXMLSerializer.SolrCoreXMLDef;
import org.apache.solr.update.SolrCoreState;
import org.apache.solr.util.DOMUtil;
import org.apache.zookeeper.KeeperException;
import org.w3c.dom.Node;
@ -155,16 +151,6 @@ class SolrCores {
}
}
protected void addCoresToList(ArrayList<SolrCoreState> coreStates) {
List<SolrCore> addCores;
synchronized (modifyLock) {
addCores = new ArrayList<SolrCore>(cores.values());
}
for (SolrCore core : addCores) {
coreStates.add(core.getUpdateHandler().getSolrCoreState());
}
}
//WARNING! This should be the _only_ place you put anything into the list of transient cores!
protected SolrCore putTransientCore(ConfigSolr cfg, String name, SolrCore core, SolrResourceLoader loader) {
SolrCore retCore;
@ -352,21 +338,10 @@ class SolrCores {
}
}
// Irrepressably ugly bit of the transition in SOLR-4196, but there as at least one test case that follows
// this path, presumably it's there for a reason.
// This is really perverse, but all we need the here is to call a couple of static methods that for back-compat
// purposes
public void persistCores(Config cfg, Properties containerProperties,
Map<String,String> rootSolrAttribs, Map<String,String> coresAttribs,
File file, File configFile, SolrResourceLoader loader) throws XPathExpressionException {
// This is expensive in the maximal case, but I think necessary. It should
// keep a reference open to all of the
// current cores while they are saved. Remember that especially the
// transient core can come and go.
//
// TODO: 5.0. remove the possibility of storing core descriptors in
// solr.xml?
//
List<SolrXMLSerializer.SolrCoreXMLDef> solrCoreXMLDefs = new ArrayList<SolrXMLSerializer.SolrCoreXMLDef>();
synchronized (modifyLock) {

354
solr/core/src/java/org/apache/solr/core/ZkContainer.java Normal file
View File

@ -0,0 +1,354 @@
package org.apache.solr.core;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.TimeoutException;
import org.apache.solr.cloud.CurrentCoreDescriptorProvider;
import org.apache.solr.cloud.SolrZkServer;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.util.SystemIdResolver;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
public class ZkContainer {
protected static Logger log = LoggerFactory.getLogger(ZkContainer.class);
/** @deprecated will be remove in Solr 5.0 (SOLR-4622) */
public static final String DEFAULT_HOST_CONTEXT = "solr";
/** @deprecated will be remove in Solr 5.0 (SOLR-4622) */
public static final String DEFAULT_HOST_PORT = "8983";
protected ZkController zkController;
private SolrZkServer zkServer;
private int zkClientTimeout;
private String hostPort;
private String hostContext;
private String host;
private String leaderVoteWait;
private int distribUpdateConnTimeout;
public SolrZkServer getZkServer() {
return zkServer;
}
public int getZkClientTimeout() {
return zkClientTimeout;
}
public String getHostPort() {
return hostPort;
}
public String getHostContext() {
return hostContext;
}
public String getHost() {
return host;
}
public String getLeaderVoteWait() {
return leaderVoteWait;
}
public int getDistribUpdateConnTimeout() {
return distribUpdateConnTimeout;
}
public int getDistribUpdateSoTimeout() {
return distribUpdateSoTimeout;
}
private int distribUpdateSoTimeout;
public ZkContainer() {
}
public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort, String hostContext, String host, String leaderVoteWait, int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
ZkController zkController = null;
// if zkHost sys property is not set, we are not using ZooKeeper
String zookeeperHost;
if(zkHost == null) {
zookeeperHost = System.getProperty("zkHost");
} else {
zookeeperHost = zkHost;
}
String zkRun = System.getProperty("zkRun");
if (zkRun == null && zookeeperHost == null)
return; // not in zk mode
this.zkClientTimeout = zkClientTimeout;
this.hostPort = hostPort;
this.hostContext = hostContext;
this.host = host;
this.leaderVoteWait = leaderVoteWait;
this.distribUpdateConnTimeout = distribUpdateConnTimeout;
this.distribUpdateSoTimeout = distribUpdateSoTimeout;
// BEGIN: SOLR-4622: deprecated hardcoded defaults for hostPort & hostContext
if (null == hostPort) {
// throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
// "'hostPort' must be configured to run SolrCloud");
log.warn("Solr 'hostPort' has not be explicitly configured, using hardcoded default of " + DEFAULT_HOST_PORT + ". This default has been deprecated and will be removed in future versions of Solr, please configure this value explicitly");
hostPort = DEFAULT_HOST_PORT;
}
if (null == hostContext) {
// throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
// "'hostContext' must be configured to run SolrCloud");
log.warn("Solr 'hostContext' has not be explicitly configured, using hardcoded default of " + DEFAULT_HOST_CONTEXT + ". This default has been deprecated and will be removed in future versions of Solr, please configure this value explicitly");
hostContext = DEFAULT_HOST_CONTEXT;
}
// END: SOLR-4622
// zookeeper in quorum mode currently causes a failure when trying to
// register log4j mbeans. See SOLR-2369
// TODO: remove after updating to an slf4j based zookeeper
System.setProperty("zookeeper.jmx.log4j.disable", "true");
if (zkRun != null) {
String zkDataHome = System.getProperty("zkServerDataDir", solrHome + "zoo_data");
String zkConfHome = System.getProperty("zkServerConfDir", solrHome);
zkServer = new SolrZkServer(zkRun, zookeeperHost, zkDataHome, zkConfHome, hostPort);
zkServer.parseConfig();
zkServer.start();
// set client from server config if not already set
if (zookeeperHost == null) {
zookeeperHost = zkServer.getClientString();
}
}
int zkClientConnectTimeout = 15000;
if (zookeeperHost != null) {
// we are ZooKeeper enabled
try {
// If this is an ensemble, allow for a long connect time for other servers to come up
if (zkRun != null && zkServer.getServers().size() > 1) {
zkClientConnectTimeout = 24 * 60 * 60 * 1000; // 1 day for embedded ensemble
log.info("Zookeeper client=" + zookeeperHost + " Waiting for a quorum.");
} else {
log.info("Zookeeper client=" + zookeeperHost);
}
String confDir = System.getProperty("bootstrap_confdir");
boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
if(!ZkController.checkChrootPath(zookeeperHost, (confDir!=null) || boostrapConf)) {
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"A chroot was specified in ZkHost but the znode doesn't exist. ");
}
zkController = new ZkController(cc, zookeeperHost, zkClientTimeout,
zkClientConnectTimeout, host, hostPort, hostContext,
leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout,
new CurrentCoreDescriptorProvider() {
@Override
public List<CoreDescriptor> getCurrentDescriptors() {
List<CoreDescriptor> descriptors = new ArrayList<CoreDescriptor>(
cc.getCoreNames().size());
Collection<SolrCore> cores = cc.getCores();
for (SolrCore core : cores) {
descriptors.add(core.getCoreDescriptor());
}
return descriptors;
}
});
if (zkRun != null && zkServer.getServers().size() > 1 && confDir == null && boostrapConf == false) {
// we are part of an ensemble and we are not uploading the config - pause to give the config time
// to get up
Thread.sleep(10000);
}
if(confDir != null) {
File dir = new File(confDir);
if(!dir.isDirectory()) {
throw new IllegalArgumentException("bootstrap_confdir must be a directory of configuration files");
}
String confName = System.getProperty(ZkController.COLLECTION_PARAM_PREFIX+ZkController.CONFIGNAME_PROP, "configuration1");
zkController.uploadConfigDir(dir, confName);
}
if(boostrapConf) {
ZkController.bootstrapConf(zkController.getZkClient(), cc.cfg, solrHome);
}
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (TimeoutException e) {
log.error("Could not connect to ZooKeeper", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (IOException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
}
}
this.zkController = zkController;
}
// Helper method to separate out creating a core from ZK as opposed to the
// "usual" way. See create()
SolrCore createFromZk(String instanceDir, CoreDescriptor dcore, SolrResourceLoader loader) {
try {
SolrResourceLoader solrLoader = null;
SolrConfig config = null;
String zkConfigName = null;
IndexSchema schema;
String collection = dcore.getCloudDescriptor().getCollectionName();
zkController.createCollectionZkNode(dcore.getCloudDescriptor());
zkConfigName = zkController.readConfigName(collection);
if (zkConfigName == null) {
log.error("Could not find config name for collection:" + collection);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"Could not find config name for collection:" + collection);
}
solrLoader = new ZkSolrResourceLoader(instanceDir, zkConfigName,
loader.getClassLoader(), ConfigSolrXml.getCoreProperties(instanceDir,
dcore), zkController);
config = getSolrConfigFromZk(zkConfigName, dcore.getConfigName(),
solrLoader);
schema = IndexSchemaFactory.buildIndexSchema(dcore.getSchemaName(),
config);
return new SolrCore(dcore.getName(), null, config, schema, dcore);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
}
}
public void registerInZk(SolrCore core) {
if (zkController != null) {
try {
zkController.register(core.getName(), core.getCoreDescriptor());
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
SolrException.log(log, "", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "",
e);
} catch (Exception e) {
// if register fails, this is really bad - close the zkController to
// minimize any damage we can cause
try {
zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
} catch (KeeperException e1) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (InterruptedException e1) {
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
}
SolrException.log(log, "", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "",
e);
}
}
}
public SolrConfig getSolrConfigFromZk(String zkConfigName, String solrConfigFileName,
SolrResourceLoader resourceLoader) {
SolrConfig cfg = null;
try {
byte[] config = zkController.getConfigFileData(zkConfigName,
solrConfigFileName);
InputSource is = new InputSource(new ByteArrayInputStream(config));
is.setSystemId(SystemIdResolver
.createSystemIdFromResourceName(solrConfigFileName));
cfg = solrConfigFileName == null ? new SolrConfig(resourceLoader,
SolrConfig.DEFAULT_CONF_FILE, is) : new SolrConfig(resourceLoader,
solrConfigFileName, is);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"getSolrConfigFromZK failed for " + zkConfigName + " "
+ solrConfigFileName, e);
}
return cfg;
}
public ZkController getZkController() {
return zkController;
}
public void publishCoresAsDown(List<SolrCore> cores) {
for (SolrCore core : cores) {
try {
zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
} catch (KeeperException e) {
CoreContainer.log.error("", e);
} catch (InterruptedException e) {
CoreContainer.log.error("", e);
}
}
}
public void close() {
if (zkController != null) {
zkController.close();
}
if (zkServer != null) {
zkServer.stop();
}
}
}

32
solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
View File

@ -91,30 +91,12 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
private void addCoreWithProps(Properties stockProps) throws Exception {
File propFile = new File(solrHomeDirectory,
stockProps.getProperty(CoreDescriptor.CORE_NAME) + File.separator + ConfigSolr.CORE_PROP_FILE);
stockProps.getProperty(CoreDescriptor.CORE_NAME) + File.separator + SolrCoreDiscoverer.CORE_PROP_FILE);
File parent = propFile.getParentFile();
assertTrue("Failed to mkdirs for " + parent.getAbsolutePath(), parent.mkdirs());
addCoreWithProps(stockProps, propFile);
}
// For testing error condition of having multiple cores with the same name.
private void addCoreWithPropsDir(String coreDir, Properties stockProps) throws Exception {
File propFile = new File(solrHomeDirectory, coreDir + File.separator + ConfigSolr.CORE_PROP_FILE);
File parent = propFile.getParentFile();
assertTrue("Failed to mkdirs for " + parent.getAbsolutePath(), parent.mkdirs());
FileOutputStream out = new FileOutputStream(propFile);
try {
stockProps.store(out, null);
} finally {
out.close();
}
addConfFiles(new File(parent, "conf"));
}
private void addConfFiles(File confDir) throws Exception {
String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf";
assertTrue("Failed to mkdirs for " + confDir.getAbsolutePath(), confDir.mkdirs());
@ -157,10 +139,10 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
try {
assertNull("defaultCore no longer allowed in solr.xml", cc.getDefaultCoreName());
assertEquals("222.333.444.555", cc.getHost());
assertEquals("6000", cc.getHostPort());
assertEquals("solrprop", cc.getHostContext());
assertEquals(20, cc.getZkClientTimeout());
assertEquals("222.333.444.555", cc.zkSys.getHost());
assertEquals("6000", cc.zkSys.getHostPort());
assertEquals("solrprop", cc.zkSys.getHostContext());
assertEquals(20, cc.zkSys.getZkClientTimeout());
TestLazyCores.checkInCores(cc, "core1");
TestLazyCores.checkNotInCores(cc, "lazy1", "core2", "collection1");
@ -200,9 +182,9 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
alt.mkdirs();
setMeUp(alt.getAbsolutePath());
addCoreWithProps(makeCorePropFile("core1", false, true, "dataDir=core1"),
new File(alt, "core1" + File.separator + ConfigSolr.CORE_PROP_FILE));
new File(alt, "core1" + File.separator + SolrCoreDiscoverer.CORE_PROP_FILE));
addCoreWithProps(makeCorePropFile("core2", false, false, "dataDir=core2"),
new File(alt, "core2" + File.separator + ConfigSolr.CORE_PROP_FILE));
new File(alt, "core2" + File.separator + SolrCoreDiscoverer.CORE_PROP_FILE));
CoreContainer cc = init();
try {
SolrCore core1 = cc.getCore("core1");

9
solr/test-framework/src/java/org/apache/solr/util/TestHarness.java
View File

@ -180,13 +180,14 @@ public class TestHarness extends BaseTestHarness {
public CoreContainer initialize() {
CoreContainer container;
try {
container = new CoreContainer(new SolrResourceLoader(SolrResourceLoader.locateSolrHome())) {
String solrHome = SolrResourceLoader.locateSolrHome();
container = new CoreContainer(new SolrResourceLoader(solrHome)) {
{
hostPort = System.getProperty("hostPort");
hostContext = "solr";
String hostPort = System.getProperty("hostPort");
String hostContext = "solr";
defaultCoreName = CoreContainer.DEFAULT_DEFAULT_CORE_NAME;
initShardHandler();
initZooKeeper(System.getProperty("zkHost"), 10000);
zkSys.initZooKeeper(this, solrHome, System.getProperty("zkHost"), 30000, hostPort, hostContext, null, "30000", 30000, 30000);
ByteArrayInputStream is = new ByteArrayInputStream(ConfigSolrXmlOld.DEF_SOLR_XML.getBytes("UTF-8"));
Config config = new Config(loader, null, new InputSource(is), null, false);
cfg = new ConfigSolrXmlOld(config, this);