Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4718 Allow solr.xml to be stored in ZooKeeper 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1514800 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erick Erickson 2013-08-16 17:16:02 +00:00
parent 3399d7ec73
commit 55142ef165
8 changed files with 298 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
solr/CHANGES.txt
View File

@ -92,6 +92,7 @@ New Features
* SOLR-5006: CREATESHARD command for 'implicit' shards (Noble Paul)
* SOLR-5017: Allow sharding based on the value of a field (Noble Paul)
* SOLR-4222: create custom sharded collection via collections API (Noble Paul)
* SOLR-4718: Allow solr.xml to be stored in ZooKeeper
* SOLR-5156: Enhance ZkCLI to allow uploading of arbitrary files to ZK.
Bug Fixes

21
solr/core/src/java/org/apache/solr/core/ConfigSolr.java
View File

@ -61,10 +61,7 @@ public abstract class ConfigSolr {
else {
inputStream = new FileInputStream(configFile);
}
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ByteStreams.copy(inputStream, baos);
String originalXml = IOUtils.toString(new ByteArrayInputStream(baos.toByteArray()), "UTF-8");
return fromInputStream(loader, new ByteArrayInputStream(baos.toByteArray()), configFile, originalXml);
return fromInputStream(loader, inputStream);
}
catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@ -76,13 +73,17 @@ public abstract class ConfigSolr {
}
public static ConfigSolr fromString(String xml) {
return fromInputStream(null, new ByteArrayInputStream(xml.getBytes(Charsets.UTF_8)), null, xml);
return fromInputStream(null, new ByteArrayInputStream(xml.getBytes(Charsets.UTF_8)));
}
public static ConfigSolr fromInputStream(SolrResourceLoader loader, InputStream is, File file, String originalXml) {
public static ConfigSolr fromInputStream(SolrResourceLoader loader, InputStream is) {
try {
Config config = new Config(loader, null, new InputSource(is), null, false);
return fromConfig(config, file, originalXml);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ByteStreams.copy(is, baos);
String originalXml = IOUtils.toString(new ByteArrayInputStream(baos.toByteArray()), "UTF-8");
ByteArrayInputStream dup = new ByteArrayInputStream(baos.toByteArray());
Config config = new Config(loader, null, new InputSource(dup), null, false);
return fromConfig(config, originalXml);
}
catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
@ -93,9 +94,9 @@ public abstract class ConfigSolr {
return fromFile(loader, new File(solrHome, SOLR_XML_FILE));
}
public static ConfigSolr fromConfig(Config config, File file, String originalXml) {
public static ConfigSolr fromConfig(Config config, String originalXml) {
boolean oldStyle = (config.getNode("solr/cores", false) != null);
return oldStyle ? new ConfigSolrXmlOld(config, file, originalXml)
return oldStyle ? new ConfigSolrXmlOld(config, originalXml)
: new ConfigSolrXml(config);
}

6
solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
View File

@ -55,15 +55,15 @@ public class ConfigSolrXmlOld extends ConfigSolr {
return "solr/cores/shardHandlerFactory";
}
public ConfigSolrXmlOld(Config config, File configFile, String originalXML) {
public ConfigSolrXmlOld(Config config, String originalXML) {
super(config);
try {
checkForIllegalConfig();
fillPropMap();
config.substituteProperties();
initCoreList();
this.persistor = isPersistent() ? new SolrXMLCoresLocator(configFile, originalXML, this)
: new SolrXMLCoresLocator.NonPersistingLocator(configFile, originalXML, this);
this.persistor = isPersistent() ? new SolrXMLCoresLocator(originalXML, this)
: new SolrXMLCoresLocator.NonPersistingLocator(originalXML, this);
}
catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);

10
solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java
View File

@ -41,7 +41,6 @@ public class SolrXMLCoresLocator implements CoresLocator {
private static final Logger logger = LoggerFactory.getLogger(SolrXMLCoresLocator.class);
private final File file;
private final String solrXmlTemplate;
private final ConfigSolrXmlOld cfg;
@ -50,13 +49,11 @@ public class SolrXMLCoresLocator implements CoresLocator {
/**
* Create a new SolrXMLCoresLocator
* @param file a File object representing the file to write out to
* @param originalXML the original content of the solr.xml file
* @param cfg the CoreContainer's config object
*/
public SolrXMLCoresLocator(File file, String originalXML, ConfigSolrXmlOld cfg) {
public SolrXMLCoresLocator(String originalXML, ConfigSolrXmlOld cfg) {
this.solrXmlTemplate = buildTemplate(originalXML);
this.file = file;
this.cfg = cfg;
}
@ -147,6 +144,7 @@ public class SolrXMLCoresLocator implements CoresLocator {
}
protected void doPersist(String xml) {
File file = new File(cfg.config.getResourceLoader().getInstanceDir(), ConfigSolr.SOLR_XML_FILE);
try {
Writer writer = new OutputStreamWriter(new FileOutputStream(file), Charsets.UTF_8);
writer.write(xml);
@ -204,8 +202,8 @@ public class SolrXMLCoresLocator implements CoresLocator {
public static class NonPersistingLocator extends SolrXMLCoresLocator {
public NonPersistingLocator(File file, String originalXML, ConfigSolrXmlOld cfg) {
super(file, originalXML, cfg);
public NonPersistingLocator(String originalXML, ConfigSolrXmlOld cfg) {
super(originalXML, cfg);
this.xml = originalXML;
}

39
solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
View File

@ -18,12 +18,14 @@
package org.apache.solr.servlet;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@ -35,9 +37,11 @@ import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.ConfigSolr;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.ContentStreamHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
@ -60,6 +64,7 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -131,12 +136,44 @@ public class SolrDispatchFilter implements Filter
log.info("SolrDispatchFilter.init() done");
}
private ConfigSolr loadConfigSolr(SolrResourceLoader loader) {
String solrxmlLocation = System.getProperty("solr.solrxml.location", "solrhome");
if (solrxmlLocation == null || "solrhome".equalsIgnoreCase(solrxmlLocation))
return ConfigSolr.fromSolrHome(loader, loader.getInstanceDir());
if ("zookeeper".equalsIgnoreCase(solrxmlLocation)) {
String zkHost = System.getProperty("zkHost");
log.info("Trying to read solr.xml from " + zkHost);
if (StringUtils.isEmpty(zkHost))
throw new SolrException(ErrorCode.SERVER_ERROR,
"Could not load solr.xml from zookeeper: zkHost system property not set");
SolrZkClient zkClient = new SolrZkClient(zkHost, 30000);
try {
if (!zkClient.exists("/solr.xml", true))
throw new SolrException(ErrorCode.SERVER_ERROR, "Could not load solr.xml from zookeeper: node not found");
byte[] data = zkClient.getData("/solr.xml", null, null, true);
return ConfigSolr.fromInputStream(loader, new ByteArrayInputStream(data));
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Could not load solr.xml from zookeeper", e);
} finally {
zkClient.close();
}
}
throw new SolrException(ErrorCode.SERVER_ERROR,
"Bad solr.solrxml.location set: " + solrxmlLocation + " - should be 'solrhome' or 'zookeeper'");
}
/**
* Override this to change CoreContainer initialization
* @return a CoreContainer to hold this server's cores
*/
protected CoreContainer createCoreContainer() {
CoreContainer cores = new CoreContainer();
SolrResourceLoader loader = new SolrResourceLoader(SolrResourceLoader.locateSolrHome());
ConfigSolr config = loadConfigSolr(loader);
CoreContainer cores = new CoreContainer(loader, config);
cores.load();
return cores;
}

233
solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java Normal file
View File

@ -0,0 +1,233 @@
package org.apache.solr.cloud;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import com.google.common.base.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.ConfigSolr;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.charset.Charset;
public class SolrXmlInZkTest extends SolrTestCaseJ4 {
@Rule
public TestRule solrTestRules = RuleChain.outerRule(new SystemPropertiesRestoreRule());
protected ZkTestServer zkServer;
protected String zkDir;
private SolrZkClient zkClient;
private ZkStateReader reader;
private static int PORT = 7000;
private ConfigSolr cfg;
@Before
public void beforeClass() {
System.setProperty("solr.solrxml.location", "zookeeper");
}
private void setUpZkAndDiskXml(boolean toZk, boolean leaveOnLocal) throws Exception {
createTempDir();
File solrHome = new File(dataDir, "home");
copyMinConf(new File(solrHome, "myCollect"));
if (leaveOnLocal) {
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr-stress-new.xml"), new File(solrHome, "solr.xml"));
}
System.setProperty("solr.solr.home", solrHome.getAbsolutePath());
ignoreException("No UpdateLog found - cannot sync");
ignoreException("No UpdateLog found - cannot recover");
System.setProperty("zkClientTimeout", "8000");
zkDir = dataDir.getAbsolutePath() + File.separator
+ "zookeeper" + System.currentTimeMillis() + "/server1/data";
zkServer = new ZkTestServer(zkDir);
zkServer.run();
System.setProperty("zkHost", zkServer.getZkAddress());
AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(),
zkServer.getZkAddress(), "solrconfig.xml", "schema.xml");
zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
if (toZk) {
zkClient.makePath("solr.xml", XML_FOR_ZK.getBytes(Charsets.UTF_8), true);
}
zkClient.close();
log.info("####SETUP_START " + getTestName());
// set some system properties for use by tests
System.setProperty("solr.test.sys.prop1", "propone");
System.setProperty("solr.test.sys.prop2", "proptwo");
Method method = SolrDispatchFilter.class.getDeclaredMethod("loadConfigSolr", SolrResourceLoader.class);
method.setAccessible(true);
Object obj = method.invoke(new SolrDispatchFilter(), new SolrResourceLoader(null));
cfg = (ConfigSolr) obj;
log.info("####SETUP_END " + getTestName());
}
private void closeZK() throws Exception {
if (zkClient != null) {
zkClient.close();
}
if (reader != null) {
reader.close();
}
zkServer.shutdown();
}
@Test
public void testXmlOnBoth() throws Exception {
try {
setUpZkAndDiskXml(true, true);
assertEquals("Should have gotten a new port the xml file sent to ZK, overrides the copy on disk",
cfg.getZkHostPort(), "9045");
} finally {
closeZK();
}
}
@Test
public void testXmlInZkOnly() throws Exception {
try {
setUpZkAndDiskXml(true, false);
assertEquals("Should have gotten a new port the xml file sent to ZK",
cfg.getZkHostPort(), "9045");
} finally {
closeZK();
}
}
@Test
public void testNotInZkAndShouldBe() throws Exception {
try {
setUpZkAndDiskXml(false, true);
fail("Should have gotten an exception here!");
} catch (InvocationTargetException ite) {
SolrException se = (SolrException) ite.getTargetException();
assertEquals("Should have an exception here, file not in ZK.",
"Could not load solr.xml from zookeeper", se.getMessage());
} finally {
closeZK();
}
}
// TODO: Solr 5.0. when we remove the default solr.xml from configSolrXmlOld this should start failing.
@Test
public void testNotInZkOrOnDisk() throws Exception {
try {
System.clearProperty("solr.solrxml.location");
System.setProperty("hostPort", "8787");
setUpZkAndDiskXml(false, false); // solr.xml not on disk either
assertEquals("Should have gotten the default port from the hard-coded default solr.xml file via sys prop.",
cfg.getZkHostPort(), "8787");
} finally {
closeZK();
}
}
@Test
public void testOnDiskOnly() throws Exception {
try {
System.clearProperty("solr.solrxml.location");
setUpZkAndDiskXml(false, true);
assertEquals("Should have gotten the default port", cfg.getZkHostPort(), "8983");
} finally {
closeZK();
}
}
@Test
public void testBadSysProp() throws Exception {
try {
System.setProperty("solr.solrxml.location", "solrHomeDir");
setUpZkAndDiskXml(false, true);
fail("Should have thrown exception in SolrXmlInZkTest.testBadSysProp");
} catch (InvocationTargetException ite) {
SolrException se = (SolrException) ite.getTargetException();
assertEquals("Should have an exception in SolrXmlInZkTest.testBadSysProp, sysprop set to bogus value.",
se.getMessage(), "Bad solr.solrxml.location set: solrHomeDir - should be 'solrhome' or 'zookeeper'");
} finally {
closeZK();
}
}
//SolrDispatchFilter.protected static ConfigSolr loadConfigSolr(SolrResourceLoader loader) {
@Test
public void testZkHostDiscovery() throws ClassNotFoundException, NoSuchMethodException,
IllegalAccessException, InstantiationException, InvocationTargetException {
// Should see an error when zkHost is not defined but solr.solrxml.location is set to zookeeper.
System.clearProperty("zkHost");
try {
Method method = SolrDispatchFilter.class.getDeclaredMethod("loadConfigSolr", SolrResourceLoader.class);
method.setAccessible(true);
method.invoke(new SolrDispatchFilter(), new SolrResourceLoader(null));
fail("Should have thrown an exception");
} catch (InvocationTargetException ite) {
assertTrue("Should be catching a SolrException", ite.getTargetException() instanceof SolrException);
String cause = ((SolrException) ite.getTargetException()).getMessage();
assertEquals("Caught Solr exception", cause,
"Could not load solr.xml from zookeeper: zkHost system property not set");
}
}
// Just a random port, I'm not going to use it but just check that the Solr instance constructed from the XML
// file in ZK overrides the default port.
private final String XML_FOR_ZK =
"<solr>" +
" <solrcloud>" +
" <str name=\"host\">127.0.0.1</str>" +
" <int name=\"hostPort\">9045</int>" +
" <str name=\"hostContext\">${hostContext:solr}</str>" +
" </solrcloud>" +
" <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">" +
" <int name=\"socketTimeout\">${socketTimeout:120000}</int>" +
" <int name=\"connTimeout\">${connTimeout:15000}</int>" +
" </shardHandlerFactory>" +
"</solr>";
}

2
solr/core/src/test/org/apache/solr/core/TestLazyCores.java
View File

@ -69,7 +69,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
FileUtils.write(solrXml, LOTS_SOLR_XML, IOUtils.CHARSET_UTF_8.toString());
ConfigSolrXmlOld config = (ConfigSolrXmlOld) ConfigSolr.fromFile(loader, solrXml);
CoresLocator locator = new SolrXMLCoresLocator.NonPersistingLocator(solrXml, LOTS_SOLR_XML, config);
CoresLocator locator = new SolrXMLCoresLocator.NonPersistingLocator(LOTS_SOLR_XML, config);
final CoreContainer cores = new CoreContainer(loader, config, locator);

15
solr/core/src/test/org/apache/solr/core/TestSolrXmlPersistor.java
View File

@ -35,7 +35,7 @@ public class TestSolrXmlPersistor {
final String solrxml = "<solr><cores adminHandler=\"/admin\"/></solr>";
SolrXMLCoresLocator persistor = new SolrXMLCoresLocator(new File("testfile.xml"), solrxml, null);
SolrXMLCoresLocator persistor = new SolrXMLCoresLocator(solrxml, null);
assertEquals(persistor.buildSolrXML(EMPTY_CD_LIST),
"<solr><cores adminHandler=\"/admin\"></cores></solr>");
@ -45,7 +45,7 @@ public class TestSolrXmlPersistor {
public void emptyCoresTagIsPersisted() {
final String solrxml = "<solr><cores adminHandler=\"/admin\"></cores></solr>";
SolrXMLCoresLocator persistor = new SolrXMLCoresLocator(new File("testfile.xml"), solrxml, null);
SolrXMLCoresLocator persistor = new SolrXMLCoresLocator(solrxml, null);
assertEquals(persistor.buildSolrXML(EMPTY_CD_LIST), "<solr><cores adminHandler=\"/admin\"></cores></solr>");
}
@ -53,7 +53,7 @@ public class TestSolrXmlPersistor {
public void emptySolrXmlIsPersisted() {
final String solrxml = "<solr></solr>";
SolrXMLCoresLocator persistor = new SolrXMLCoresLocator(new File("testfile.xml"), solrxml, null);
SolrXMLCoresLocator persistor = new SolrXMLCoresLocator(solrxml, null);
assertEquals(persistor.buildSolrXML(EMPTY_CD_LIST), "<solr><cores></cores></solr>");
}
@ -68,7 +68,7 @@ public class TestSolrXmlPersistor {
final CoreDescriptor cd = new CoreDescriptor(cc, "testcore", "instance/dir/");
List<CoreDescriptor> cds = ImmutableList.of(cd);
SolrXMLCoresLocator persistor = new SolrXMLCoresLocator(new File("testfile.xml"), solrxml, null);
SolrXMLCoresLocator persistor = new SolrXMLCoresLocator(solrxml, null);
assertEquals(persistor.buildSolrXML(cds),
"<solr><cores>" + SolrXMLCoresLocator.NEWLINE
+ " <core name=\"testcore\" instanceDir=\"instance/dir/\"/>" + SolrXMLCoresLocator.NEWLINE
@ -89,7 +89,7 @@ public class TestSolrXmlPersistor {
"</cores>" +
"</solr>";
SolrXMLCoresLocator locator = new SolrXMLCoresLocator(new File("testfile.xml"), solrxml, null);
SolrXMLCoresLocator locator = new SolrXMLCoresLocator(solrxml, null);
assertTrue(locator.getTemplate().contains("{{CORES_PLACEHOLDER}}"));
assertTrue(locator.getTemplate().contains("<shardHandlerFactory "));
assertTrue(locator.getTemplate().contains("${socketTimeout:500}"));
@ -107,15 +107,14 @@ public class TestSolrXmlPersistor {
"</cores>" +
"</solr>";
SolrXMLCoresLocator locator = new SolrXMLCoresLocator(new File("testfile.xml"), solrxml, null);
SolrXMLCoresLocator locator = new SolrXMLCoresLocator(solrxml, null);
assertTrue(locator.getTemplate().contains("{{CORES_PLACEHOLDER}}"));
assertTrue(locator.getTemplate().contains("<shardHandlerFactory "));
}
@Test
public void complexXmlIsParsed() {
SolrXMLCoresLocator locator = new SolrXMLCoresLocator(new File("testfile.xml"),
TestSolrXmlPersistence.SOLR_XML_LOTS_SYSVARS, null);
SolrXMLCoresLocator locator = new SolrXMLCoresLocator(TestSolrXmlPersistence.SOLR_XML_LOTS_SYSVARS, null);
assertTrue(locator.getTemplate().contains("{{CORES_PLACEHOLDER}}"));
}