From 79f996a31bd597c8416a43802f9c9924ab6a26dd Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Fri, 26 Apr 2013 14:32:35 +0000 Subject: [PATCH] SOLR-4757: Change the example to use the new solr.xml format and core discovery by directory structure. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1476227 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 ++ .../src/java/org/apache/solr/cloud/ZkCLI.java | 2 +- .../org/apache/solr/cloud/ZkController.java | 5 +- .../org/apache/solr/core/ConfigSolrXml.java | 6 +-- .../org/apache/solr/core/CoreContainer.java | 3 ++ .../org/apache/solr/core/CoreDescriptor.java | 3 +- .../apache/solr/core/SolrCoreDiscoverer.java | 7 +-- .../test-files/old-solr-example/README.txt | 1 + .../src/test-files/old-solr-example/solr.xml | 53 +++++++++++++++++++ .../test/org/apache/solr/cloud/ZkCLITest.java | 19 ++++++- solr/example/solr/collection1/core.properties | 1 + solr/example/solr/solr.xml | 37 +++++-------- .../client/solrj/SolrExampleBinaryTest.java | 2 +- .../solr/client/solrj/SolrExampleXMLTest.java | 2 +- .../embedded/SolrExampleStreamingTest.java | 4 +- .../solrj/impl/BasicHttpSolrServerTest.java | 8 +-- .../org/apache/solr/SolrJettyTestBase.java | 18 +++---- 17 files changed, 120 insertions(+), 54 deletions(-) create mode 100644 solr/core/src/test-files/old-solr-example/README.txt create mode 100644 solr/core/src/test-files/old-solr-example/solr.xml create mode 100644 solr/example/solr/collection1/core.properties diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 31dbdb55252..b24de707bd7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -94,6 +94,9 @@ Other Changes * SOLR-4547: Move logging of filenames on commit from INFO to DEBUG. (Shawn Heisey, hossman) +* SOLR-4757: Change the example to use the new solr.xml format and core + discovery by directory structure. (Mark Miller) + ================== 4.3.0 ================== Versions of Major Components diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java index 4cd131a2ce6..f8e171efab2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java @@ -187,7 +187,7 @@ public class ZkCLI { Config config = new Config(loader, null, new InputSource(is), null, false); boolean oldStyle = (config.getNode("solr/cores", false) != null); - // cfg = new ConfigSolrXml(loader, null, is, null, false, this); + if (oldStyle) { cfg = new ConfigSolrXmlOld(config, null); } else { diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index ec872860448..0eb0a4d7784 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -1441,12 +1441,15 @@ public final class ZkController { public static void bootstrapConf(SolrZkClient zkClient, ConfigSolr cfg, String solrHome) throws IOException, KeeperException, InterruptedException { - log.info("bootstraping config into ZooKeeper using solr.xml"); List allCoreNames = cfg.getAllCoreNames(); + + log.info("bootstraping config for " + allCoreNames.size() + " cores into ZooKeeper using solr.xml from " + solrHome); + for (String coreName : allCoreNames) { String rawName = PropertiesUtil.substituteProperty(cfg.getProperty(coreName, "name", null), new Properties()); String instanceDir = cfg.getProperty(coreName, "instanceDir", null); File idir = new File(instanceDir); + System.out.println("idir:" + idir); if (!idir.isAbsolute()) { idir = new File(solrHome, instanceDir); } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java b/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java index 880fe80cfdd..0c274aa01bc 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java @@ -49,15 +49,15 @@ public class ConfigSolrXml extends ConfigSolr { public ConfigSolrXml(Config config, CoreContainer container) throws ParserConfigurationException, IOException, SAXException { super(config); - checkForIllegalConfig(container); + checkForIllegalConfig(); fillPropMap(); - String coreRoot = get(CfgProp.SOLR_COREROOTDIRECTORY, (container == null ? null : container.getSolrHome())); + String coreRoot = get(CfgProp.SOLR_COREROOTDIRECTORY, (container == null ? config.getResourceLoader().getInstanceDir() : container.getSolrHome())); coreDescriptorMap = solrCoreDiscoverer.discover(container, new File(coreRoot)); } - private void checkForIllegalConfig(CoreContainer container) throws IOException { + private void checkForIllegalConfig() throws IOException { // Do sanity checks - we don't want to find old style config failIfFound("solr/@coreLoadThreads"); diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index ac8d92d3f9a..aca2d4ef0a5 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -183,6 +183,9 @@ public class CoreContainer log.info("looking for solr config file: " + fconf.getAbsolutePath()); cores = new CoreContainer(solrHome); + // first we find zkhost, then we check for solr.xml in zk + // 1. look for zkhost from sys prop 2. look for zkhost in {solr.home}/solr.properties + // Either we have a config file or not. if (fconf.exists()) { diff --git a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java index 1c221fb0c60..dc42278fd24 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java +++ b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java @@ -78,6 +78,7 @@ public class CoreDescriptor { coreProperties.put(CORE_TRANSIENT, "false"); } + public CoreDescriptor(CoreContainer container, String name, String instanceDir) { this(container); doInit(name, instanceDir); @@ -202,7 +203,7 @@ public class CoreDescriptor { */ public String getInstanceDir() { String instDir = coreProperties.getProperty(CORE_INSTDIR); - if (instDir == null) return null; // No worse than before. + if (instDir == null) return null; if (new File(instDir).isAbsolute()) { return SolrResourceLoader.normalizeDir( diff --git a/solr/core/src/java/org/apache/solr/core/SolrCoreDiscoverer.java b/solr/core/src/java/org/apache/solr/core/SolrCoreDiscoverer.java index 48987e8d482..512f540a7d3 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCoreDiscoverer.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCoreDiscoverer.java @@ -37,9 +37,9 @@ public class SolrCoreDiscoverer { public Map discover(CoreContainer container, File root) throws IOException { Map coreDescriptorMap = new HashMap(); - if (container != null) { - walkFromHere(root, container, coreDescriptorMap); - } + + walkFromHere(root, container, coreDescriptorMap); + return coreDescriptorMap; } @@ -83,6 +83,7 @@ public class SolrCoreDiscoverer { // Too much of the code depends on this value being here, but it is NOT supported in discovery mode, so // ignore it if present in the core.properties file. + System.out.println("SET INST DIR:" + childFile.getPath()); props.setProperty(CoreDescriptor.CORE_INSTDIR, childFile.getPath()); if (props.getProperty(CoreDescriptor.CORE_NAME) == null) { diff --git a/solr/core/src/test-files/old-solr-example/README.txt b/solr/core/src/test-files/old-solr-example/README.txt new file mode 100644 index 00000000000..6242cff237b --- /dev/null +++ b/solr/core/src/test-files/old-solr-example/README.txt @@ -0,0 +1 @@ +This is around for back compat testing purposes and should be able to be removed in Solr 5.0 \ No newline at end of file diff --git a/solr/core/src/test-files/old-solr-example/solr.xml b/solr/core/src/test-files/old-solr-example/solr.xml new file mode 100644 index 00000000000..75da88a52f1 --- /dev/null +++ b/solr/core/src/test-files/old-solr-example/solr.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java index e6ee995fdd2..70c4ad4bc03 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java @@ -20,6 +20,7 @@ package org.apache.solr.cloud; import java.io.File; import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; @@ -42,6 +43,8 @@ public class ZkCLITest extends SolrTestCaseJ4 { protected ZkTestServer zkServer; protected String zkDir; + + private String solrHome; private SolrZkClient zkClient; @@ -62,6 +65,18 @@ public class ZkCLITest extends SolrTestCaseJ4 { log.info("####SETUP_START " + getTestName()); createTempDir(); + boolean useNewSolrXml = random().nextBoolean(); + + if (useNewSolrXml) { + solrHome = ExternalPaths.EXAMPLE_HOME; + } else { + File tmpSolrHome = new File(dataDir, "tmp-solr-home"); + FileUtils.copyDirectory(new File(ExternalPaths.EXAMPLE_HOME), tmpSolrHome); + FileUtils.copyFile(new File(ExternalPaths.SOURCE_HOME, "core/src/test-files/old-solr-example/solr.xml"), new File(tmpSolrHome, "solr.xml")); + solrHome = tmpSolrHome.getAbsolutePath(); + } + + zkDir = dataDir.getAbsolutePath() + File.separator + "zookeeper/server1/data"; log.info("ZooKeeper dataDir:" + zkDir); @@ -83,7 +98,7 @@ public class ZkCLITest extends SolrTestCaseJ4 { public void testBootstrap() throws Exception { // test bootstrap_conf String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", - "bootstrap", "-solrhome", ExternalPaths.EXAMPLE_HOME}; + "bootstrap", "-solrhome", this.solrHome}; ZkCLI.main(args); assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/collection1", true)); @@ -102,7 +117,7 @@ public class ZkCLITest extends SolrTestCaseJ4 { assertFalse(zkClient.exists(chroot, true)); String[] args = new String[] {"-zkhost", zkServer.getZkAddress() + chroot, - "-cmd", "bootstrap", "-solrhome", ExternalPaths.EXAMPLE_HOME}; + "-cmd", "bootstrap", "-solrhome", this.solrHome}; ZkCLI.main(args); diff --git a/solr/example/solr/collection1/core.properties b/solr/example/solr/collection1/core.properties new file mode 100644 index 00000000000..bc0cf7d0db9 --- /dev/null +++ b/solr/example/solr/collection1/core.properties @@ -0,0 +1 @@ +name=collection1 \ No newline at end of file diff --git a/solr/example/solr/solr.xml b/solr/example/solr/solr.xml index 75da88a52f1..d02800b622a 100644 --- a/solr/example/solr/solr.xml +++ b/solr/example/solr/solr.xml @@ -26,28 +26,19 @@ http://wiki.apache.org/solr/CoreAdmin --> - - - + + + + ${host:} + ${jetty.port:8983} + ${hostContext:solr} + ${zkClientTimeout:15000} + + + + ${socketTimeout:0} + ${connTimeout:0} + - - - - diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.java index 3c846b20c52..9cb548bf4f1 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryTest.java @@ -41,7 +41,7 @@ public class SolrExampleBinaryTest extends SolrExampleTests { { try { // setup the server... - String url = jetty.getBaseUrl().toString(); + String url = jetty.getBaseUrl().toString() + "/collection1"; HttpSolrServer s = new HttpSolrServer( url ); s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); s.setDefaultMaxConnectionsPerHost(100); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleXMLTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleXMLTest.java index eb81b6d27e6..e161f94458b 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleXMLTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleXMLTest.java @@ -36,7 +36,7 @@ public class SolrExampleXMLTest extends SolrExampleTests { @Override public SolrServer createNewSolrServer() { try { - String url = jetty.getBaseUrl().toString(); + String url = jetty.getBaseUrl().toString() + "/collection1"; HttpSolrServer s = new HttpSolrServer(url); s.setUseMultiPartPost(random().nextBoolean()); s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.java index 9fe6760d0b9..35abddeefba 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingTest.java @@ -55,7 +55,7 @@ public class SolrExampleStreamingTest extends SolrExampleTests { { try { // setup the server... - String url = jetty.getBaseUrl().toString(); + String url = jetty.getBaseUrl().toString() + "/collection1"; // smaller queue size hits locks more often ConcurrentUpdateSolrServer s = new ConcurrentUpdateSolrServer( url, 2, 5 ) { @@ -80,7 +80,7 @@ public class SolrExampleStreamingTest extends SolrExampleTests { // SOLR-3903 final List failures = new ArrayList(); ConcurrentUpdateSolrServer s = new ConcurrentUpdateSolrServer - (jetty.getBaseUrl().toString(), 2, 2) { + (jetty.getBaseUrl().toString() + "/collection1", 2, 2) { @Override public void handleError(Throwable ex) { failures.add(ex); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java index 4e30ee287fa..9f2af65ea5f 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java @@ -57,7 +57,7 @@ public class BasicHttpSolrServerTest extends SolrJettyTestBase { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - resp.sendRedirect("/solr/select?" + req.getQueryString()); + resp.sendRedirect("/solr/collection1/select?" + req.getQueryString()); } } @@ -439,7 +439,7 @@ public class BasicHttpSolrServerTest extends SolrJettyTestBase { assertNull(DebugServlet.headers.get("Accept-Encoding")); // verify server compresses output - HttpGet get = new HttpGet(jetty.getBaseUrl().toString() + + HttpGet get = new HttpGet(jetty.getBaseUrl().toString() + "/collection1" + "/select?q=foo&wt=xml"); get.setHeader("Accept-Encoding", "gzip"); HttpClient client = HttpClientUtil.createClient(null); @@ -458,7 +458,7 @@ public class BasicHttpSolrServerTest extends SolrJettyTestBase { } // verify compressed response can be handled - server = new HttpSolrServer(jetty.getBaseUrl().toString()); + server = new HttpSolrServer(jetty.getBaseUrl().toString() + "/collection1"); server.setAllowCompression(true); q = new SolrQuery("foo"); QueryResponse response = server.query(q); @@ -485,7 +485,7 @@ public class BasicHttpSolrServerTest extends SolrJettyTestBase { @Test public void testGetRawStream() throws SolrServerException, IOException{ HttpClient client = HttpClientUtil.createClient(null); - HttpSolrServer server = new HttpSolrServer(jetty.getBaseUrl().toString(), + HttpSolrServer server = new HttpSolrServer(jetty.getBaseUrl().toString() + "/collection1", client, null); QueryRequest req = new QueryRequest(); NamedList response = server.request(req); diff --git a/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java b/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java index e737cee7c02..3d619c46da9 100755 --- a/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java @@ -17,25 +17,19 @@ package org.apache.solr; * limitations under the License. */ +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; + import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.util.ExternalPaths; - -import java.io.File; -import java.util.Collections; -import java.util.Map; -import java.util.HashMap; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.apache.solr.util.RESTfulServerProvider; -import org.apache.solr.util.RestTestHarness; import org.eclipse.jetty.servlet.ServletHolder; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.restlet.ext.servlet.ServerServlet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -159,7 +153,7 @@ abstract public class SolrJettyTestBase extends SolrTestCaseJ4 if (jetty != null) { try { // setup the server... - String url = jetty.getBaseUrl().toString(); + String url = jetty.getBaseUrl().toString() + "/" + "collection1"; HttpSolrServer s = new HttpSolrServer( url ); s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); s.setDefaultMaxConnectionsPerHost(100);