SOLR-4028: When using ZK chroot, it would be nice if Solr would create the initial path when it doesn't exist.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1419866 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2012-12-10 23:18:14 +00:00
parent ef21415d6c
commit 89bddae466
6 changed files with 254 additions and 4 deletions

View File

@ -124,6 +124,9 @@ New Features
* SOLR-4124: You should be able to set the update log directory with the
CoreAdmin API the same way as the data directory. (Mark Miller)
* SOLR-4028: When using ZK chroot, it would be nice if Solr would create the
initial path when it doesn't exist. (Tomas Fernandez Lobbe via Mark Miller)
Optimizations
----------------------

View File

@ -173,6 +173,12 @@ public class ZkCLI {
InputSource cfgis = new InputSource(new File(solrHome, SOLR_XML)
.toURI().toASCIIString());
Config cfg = new Config(loader, null, cfgis, null, false);
if(!ZkController.checkChrootPath(zkServerAddress, true)) {
System.out.println("A chroot was specified in zkHost but the znode doesn't exist. ");
System.exit(1);
}
ZkController.bootstrapConf(zkClient, cfg, solrHome);
} else if (line.getOptionValue(CMD).equals(UPCONFIG)) {
@ -184,6 +190,11 @@ public class ZkCLI {
String confDir = line.getOptionValue(CONFDIR);
String confName = line.getOptionValue(CONFNAME);
if(!ZkController.checkChrootPath(zkServerAddress, true)) {
System.out.println("A chroot was specified in zkHost but the znode doesn't exist. ");
System.exit(1);
}
ZkController.uploadConfigDir(zkClient, new File(confDir), confName);
} else if (line.getOptionValue(CMD).equals(DOWNCONFIG)) {
if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {

View File

@ -462,6 +462,37 @@ public final class ZkController {
}
/**
* Validates if the chroot exists in zk (or if it is successfully created). Optionally, if create is set to true this method will create the path
* in case it doesn't exist
* @return true if the path exists or is created
* false if the path doesn't exist and 'create' = false
*/
public static boolean checkChrootPath(String zkHost, boolean create) throws KeeperException, InterruptedException {
if(!containsChroot(zkHost)) {
return true;
}
log.info("zkHost includes chroot");
String chrootPath = zkHost.substring(zkHost.indexOf("/"), zkHost.length());
SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0, zkHost.indexOf("/")), 60*1000);
boolean exists = tmpClient.exists(chrootPath, true);
if(!exists && create) {
tmpClient.makePath(chrootPath, false, true);
exists = true;
}
tmpClient.close();
return exists;
}
/**
* Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions
*/
private static boolean containsChroot(String zkHost) {
return zkHost.contains("/");
}
public boolean isConnected() {
return zkClient.isConnected();
}

View File

@ -242,6 +242,14 @@ public class CoreContainer
} else {
log.info("Zookeeper client=" + zookeeperHost);
}
String confDir = System.getProperty("bootstrap_confdir");
boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
if(!ZkController.checkChrootPath(zookeeperHost, (confDir!=null) || boostrapConf)) {
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"A chroot was specified in ZkHost but the znode doesn't exist. ");
}
zkController = new ZkController(this, zookeeperHost, zkClientTimeout, zkClientConnectTimeout, host, hostPort, hostContext, leaderVoteWait, new CurrentCoreDescriptorProvider() {
@Override
@ -254,8 +262,7 @@ public class CoreContainer
}
});
String confDir = System.getProperty("bootstrap_confdir");
boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
if (zkRun != null && zkServer.getServers().size() > 1 && confDir == null && boostrapConf == false) {
// we are part of an ensemble and we are not uploading the config - pause to give the config time

View File

@ -0,0 +1,186 @@
package org.apache.solr.cloud;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.util.ExternalPaths;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestZkChroot extends SolrTestCaseJ4 {
protected static Logger log = LoggerFactory.getLogger(TestZkChroot.class);
protected CoreContainer cores = null;
private String home;
protected ZkTestServer zkServer;
protected String zkDir;
@Override
@Before
public void setUp() throws Exception {
super.setUp();
createTempDir();
zkDir = dataDir.getAbsolutePath() + File.separator
+ "zookeeper/server1/data";
zkServer = new ZkTestServer(zkDir);
zkServer.run();
home = ExternalPaths.EXAMPLE_HOME;
}
@Override
@After
public void tearDown() throws Exception {
System.clearProperty("zkHost");
if (cores != null) {
cores.shutdown();
cores = null;
}
zkServer.shutdown();
String skip = System.getProperty("solr.test.leavedatadir");
if (null != skip && 0 != skip.trim().length()) {
log.info("NOTE: per solr.test.leavedatadir, dataDir will not be removed: "
+ dataDir.getAbsolutePath());
} else {
if (!AbstractSolrTestCase.recurseDelete(dataDir)) {
log.warn("!!!! WARNING: best effort to remove "
+ dataDir.getAbsolutePath() + " FAILED !!!!!");
}
}
zkServer = null;
zkDir = null;
super.tearDown();
}
@Test
public void testChrootBootstrap() throws Exception {
String chroot = "/foo/bar";
System.setProperty("bootstrap_conf", "true");
System.setProperty("zkHost", zkServer.getZkHost() + chroot);
SolrZkClient zkClient = null;
SolrZkClient zkClient2 = null;
try {
cores = new CoreContainer(home, new File(home, "solr.xml"));
zkClient = cores.getZkController().getZkClient();
assertTrue(zkClient.exists("/clusterstate.json", true));
assertFalse(zkClient.exists(chroot + "/clusterstate.json", true));
zkClient2 = new SolrZkClient(zkServer.getZkHost(),
AbstractZkTestCase.TIMEOUT);
assertTrue(zkClient2.exists(chroot + "/clusterstate.json", true));
assertFalse(zkClient2.exists("/clusterstate.json", true));
} finally {
if (cores != null) cores.shutdown();
if (zkClient != null) zkClient.close();
if (zkClient2 != null) zkClient2.close();
}
}
@Test
public void testNoBootstrapConf() throws Exception {
String chroot = "/foo/bar2";
System.setProperty("bootstrap_conf", "false");
System.setProperty("zkHost", zkServer.getZkHost() + chroot);
SolrZkClient zkClient = null;
try {
zkClient = new SolrZkClient(zkServer.getZkHost(),
AbstractZkTestCase.TIMEOUT);
assertFalse("Path '" + chroot + "' should not exist before the test",
zkClient.exists(chroot, true));
cores = new CoreContainer(home, new File(home, "solr.xml"));
fail("There should be a zk exception, as the initial path doesn't exist");
} catch (ZooKeeperException e) {
// expected
assertFalse("Path shouldn't have been created",
zkClient.exists(chroot, true));// check the path was not created
} finally {
if (cores != null) cores.shutdown();
if (zkClient != null) zkClient.close();
}
}
@Test
public void testWithUploadDir() throws Exception {
String chroot = "/foo/bar3";
String configName = "testWithUploadDir";
System.setProperty("bootstrap_conf", "false");
System.setProperty("bootstrap_confdir", home + "/collection1/conf");
System.setProperty("collection.configName", configName);
System.setProperty("zkHost", zkServer.getZkHost() + chroot);
SolrZkClient zkClient = null;
try {
zkClient = new SolrZkClient(zkServer.getZkHost(),
AbstractZkTestCase.TIMEOUT);
assertFalse("Path '" + chroot + "' should not exist before the test",
zkClient.exists(chroot, true));
cores = new CoreContainer(home, new File(home, "solr.xml"));
assertTrue(
"solrconfig.xml should have been uploaded to zk to the correct config directory",
zkClient.exists(chroot + ZkController.CONFIGS_ZKNODE + "/"
+ configName + "/solrconfig.xml", true));
} finally {
if (cores != null) cores.shutdown();
if (zkClient != null) zkClient.close();
}
}
@Test
public void testInitPathExists() throws Exception {
String chroot = "/foo/bar4";
System.setProperty("bootstrap_conf", "true");
System.setProperty("zkHost", zkServer.getZkHost() + chroot);
SolrZkClient zkClient = null;
try {
zkClient = new SolrZkClient(zkServer.getZkHost(),
AbstractZkTestCase.TIMEOUT);
zkClient.makePath("/foo/bar4", true);
assertTrue(zkClient.exists(chroot, true));
assertFalse(zkClient.exists(chroot + "/clusterstate.json", true));
cores = new CoreContainer(home, new File(home, "solr.xml"));
assertTrue(zkClient.exists(chroot + "/clusterstate.json", true));
} finally {
if (cores != null) cores.shutdown();
if (zkClient != null) zkClient.close();
}
}
}

View File

@ -94,10 +94,22 @@ public class ZkCLITest extends SolrTestCaseJ4 {
assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/core0", true));
assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/core1", true));
}
@Test
public void testBootstrapWithChroot() throws Exception {
String chroot = "/foo/bar";
assertFalse(zkClient.exists(chroot, true));
String[] args = new String[] {"-zkhost", zkServer.getZkAddress() + chroot,
"-cmd", "bootstrap", "-solrhome", ExternalPaths.EXAMPLE_HOME};
ZkCLI.main(args);
assertTrue(zkClient.exists(chroot + ZkController.CONFIGS_ZKNODE
+ "/collection1", true));
}
@Test
public void testMakePath() throws Exception {
// test bootstrap_conf