SOLR-4028: When using ZK chroot, it would be nice if Solr would create the initial path when it doesn't exist.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1419866 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -125,6 +125,9 @@ New Features
 * SOLR-4124: You should be able to set the update log directory with the
   CoreAdmin API the same way as the data directory. (Mark Miller)
   
+* SOLR-4028: When using ZK chroot, it would be nice if Solr would create the 
+  initial path when it doesn't exist. (Tomas Fernandez Lobbe via Mark Miller)
+
 Optimizations
 ----------------------
 

solr/core/src/java/org/apache/solr/cloud/ZkCLI.java

@@ -173,6 +173,12 @@ public class ZkCLI {
           InputSource cfgis = new InputSource(new File(solrHome, SOLR_XML)
               .toURI().toASCIIString());
           Config cfg = new Config(loader, null, cfgis, null, false);
+          
+          if(!ZkController.checkChrootPath(zkServerAddress, true)) {
+            System.out.println("A chroot was specified in zkHost but the znode doesn't exist. ");
+            System.exit(1);
+          }
+          
           ZkController.bootstrapConf(zkClient, cfg, solrHome);
           
         } else if (line.getOptionValue(CMD).equals(UPCONFIG)) {
@@ -184,6 +190,11 @@ public class ZkCLI {
           String confDir = line.getOptionValue(CONFDIR);
           String confName = line.getOptionValue(CONFNAME);
           
+          if(!ZkController.checkChrootPath(zkServerAddress, true)) {
+            System.out.println("A chroot was specified in zkHost but the znode doesn't exist. ");
+            System.exit(1);
+          }
+          
           ZkController.uploadConfigDir(zkClient, new File(confDir), confName);
         } else if (line.getOptionValue(CMD).equals(DOWNCONFIG)) {
           if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {

solr/core/src/java/org/apache/solr/cloud/ZkController.java

@@ -462,6 +462,37 @@ public final class ZkController {
 
   }
 
+  /**
+   * Validates if the chroot exists in zk (or if it is successfully created). Optionally, if create is set to true this method will create the path
+   * in case it doesn't exist
+   * @return true if the path exists or is created
+   * false if the path doesn't exist and 'create' = false
+   */
+  public static boolean checkChrootPath(String zkHost, boolean create) throws KeeperException, InterruptedException {
+    if(!containsChroot(zkHost)) {
+      return true;
+    }
+    log.info("zkHost includes chroot");
+    String chrootPath = zkHost.substring(zkHost.indexOf("/"), zkHost.length());
+    SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0, zkHost.indexOf("/")), 60*1000);
+    boolean exists = tmpClient.exists(chrootPath, true);
+    if(!exists && create) {
+      tmpClient.makePath(chrootPath, false, true);
+      exists = true;
+    }
+    tmpClient.close();
+    return exists;
+  }
+
+
+  /**
+   * Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions
+   */
+  private static boolean containsChroot(String zkHost) {
+    return zkHost.contains("/");
+  }
+
+
   public boolean isConnected() {
     return zkClient.isConnected();
   }

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -242,6 +242,14 @@ public class CoreContainer
         } else {
           log.info("Zookeeper client=" + zookeeperHost);          
         }
+        String confDir = System.getProperty("bootstrap_confdir");
+        boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");  
+        
+        if(!ZkController.checkChrootPath(zookeeperHost, (confDir!=null) || boostrapConf)) {
+          throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+              "A chroot was specified in ZkHost but the znode doesn't exist. ");
+        }
+        
         zkController = new ZkController(this, zookeeperHost, zkClientTimeout, zkClientConnectTimeout, host, hostPort, hostContext, leaderVoteWait, new CurrentCoreDescriptorProvider() {
           
           @Override
@@ -254,8 +262,7 @@ public class CoreContainer
           }
         });        
 
-        String confDir = System.getProperty("bootstrap_confdir");
-        boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
+        
         
         if (zkRun != null && zkServer.getServers().size() > 1 && confDir == null && boostrapConf == false) {
           // we are part of an ensemble and we are not uploading the config - pause to give the config time

solr/core/src/test/org/apache/solr/cloud/TestZkChroot.java

@@ -0,0 +1,186 @@
+package org.apache.solr.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestZkChroot extends SolrTestCaseJ4 {
+  protected static Logger log = LoggerFactory.getLogger(TestZkChroot.class);
+  protected CoreContainer cores = null;
+  private String home;
+  
+  protected ZkTestServer zkServer;
+  protected String zkDir;
+  
+  @Override
+  @Before
+  public void setUp() throws Exception {
+    super.setUp();
+    createTempDir();
+    zkDir = dataDir.getAbsolutePath() + File.separator
+        + "zookeeper/server1/data";
+    zkServer = new ZkTestServer(zkDir);
+    zkServer.run();
+    home = ExternalPaths.EXAMPLE_HOME;
+    
+  }
+  
+  @Override
+  @After
+  public void tearDown() throws Exception {
+    System.clearProperty("zkHost");
+    
+    if (cores != null) {
+      cores.shutdown();
+      cores = null;
+    }
+    
+    zkServer.shutdown();
+    
+    String skip = System.getProperty("solr.test.leavedatadir");
+    if (null != skip && 0 != skip.trim().length()) {
+      log.info("NOTE: per solr.test.leavedatadir, dataDir will not be removed: "
+          + dataDir.getAbsolutePath());
+    } else {
+      if (!AbstractSolrTestCase.recurseDelete(dataDir)) {
+        log.warn("!!!! WARNING: best effort to remove "
+            + dataDir.getAbsolutePath() + " FAILED !!!!!");
+      }
+    }
+    
+    zkServer = null;
+    zkDir = null;
+    
+    super.tearDown();
+  }
+  
+  @Test
+  public void testChrootBootstrap() throws Exception {
+    String chroot = "/foo/bar";
+    
+    System.setProperty("bootstrap_conf", "true");
+    System.setProperty("zkHost", zkServer.getZkHost() + chroot);
+    SolrZkClient zkClient = null;
+    SolrZkClient zkClient2 = null;
+    
+    try {
+      cores = new CoreContainer(home, new File(home, "solr.xml"));
+      zkClient = cores.getZkController().getZkClient();
+      
+      assertTrue(zkClient.exists("/clusterstate.json", true));
+      assertFalse(zkClient.exists(chroot + "/clusterstate.json", true));
+      
+      zkClient2 = new SolrZkClient(zkServer.getZkHost(),
+          AbstractZkTestCase.TIMEOUT);
+      assertTrue(zkClient2.exists(chroot + "/clusterstate.json", true));
+      assertFalse(zkClient2.exists("/clusterstate.json", true));
+    } finally {
+      if (cores != null) cores.shutdown();
+      if (zkClient != null) zkClient.close();
+      if (zkClient2 != null) zkClient2.close();
+    }
+  }
+  
+  @Test
+  public void testNoBootstrapConf() throws Exception {
+    String chroot = "/foo/bar2";
+    
+    System.setProperty("bootstrap_conf", "false");
+    System.setProperty("zkHost", zkServer.getZkHost() + chroot);
+    
+    SolrZkClient zkClient = null;
+    
+    try {
+      zkClient = new SolrZkClient(zkServer.getZkHost(),
+          AbstractZkTestCase.TIMEOUT);
+      assertFalse("Path '" + chroot + "' should not exist before the test",
+          zkClient.exists(chroot, true));
+      cores = new CoreContainer(home, new File(home, "solr.xml"));
+      fail("There should be a zk exception, as the initial path doesn't exist");
+    } catch (ZooKeeperException e) {
+      // expected
+      assertFalse("Path shouldn't have been created",
+          zkClient.exists(chroot, true));// check the path was not created
+    } finally {
+      if (cores != null) cores.shutdown();
+      if (zkClient != null) zkClient.close();
+    }
+  }
+  
+  @Test
+  public void testWithUploadDir() throws Exception {
+    String chroot = "/foo/bar3";
+    String configName = "testWithUploadDir";
+    
+    System.setProperty("bootstrap_conf", "false");
+    System.setProperty("bootstrap_confdir", home + "/collection1/conf");
+    System.setProperty("collection.configName", configName);
+    System.setProperty("zkHost", zkServer.getZkHost() + chroot);
+    SolrZkClient zkClient = null;
+    
+    try {
+      zkClient = new SolrZkClient(zkServer.getZkHost(),
+          AbstractZkTestCase.TIMEOUT);
+      assertFalse("Path '" + chroot + "' should not exist before the test",
+          zkClient.exists(chroot, true));
+      cores = new CoreContainer(home, new File(home, "solr.xml"));
+      assertTrue(
+          "solrconfig.xml should have been uploaded to zk to the correct config directory",
+          zkClient.exists(chroot + ZkController.CONFIGS_ZKNODE + "/"
+              + configName + "/solrconfig.xml", true));
+    } finally {
+      if (cores != null) cores.shutdown();
+      if (zkClient != null) zkClient.close();
+    }
+  }
+  
+  @Test
+  public void testInitPathExists() throws Exception {
+    String chroot = "/foo/bar4";
+    
+    System.setProperty("bootstrap_conf", "true");
+    System.setProperty("zkHost", zkServer.getZkHost() + chroot);
+    SolrZkClient zkClient = null;
+    
+    try {
+      zkClient = new SolrZkClient(zkServer.getZkHost(),
+          AbstractZkTestCase.TIMEOUT);
+      zkClient.makePath("/foo/bar4", true);
+      assertTrue(zkClient.exists(chroot, true));
+      assertFalse(zkClient.exists(chroot + "/clusterstate.json", true));
+      
+      cores = new CoreContainer(home, new File(home, "solr.xml"));
+      assertTrue(zkClient.exists(chroot + "/clusterstate.json", true));
+    } finally {
+      if (cores != null) cores.shutdown();
+      if (zkClient != null) zkClient.close();
+    }
+  }
+}

solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java

@@ -94,8 +94,20 @@ public class ZkCLITest extends SolrTestCaseJ4 {
     
     assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/core0", true));
     assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/core1", true));
+  }
   
+  @Test
+  public void testBootstrapWithChroot() throws Exception {
+    String chroot = "/foo/bar";
+    assertFalse(zkClient.exists(chroot, true));
     
+    String[] args = new String[] {"-zkhost", zkServer.getZkAddress() + chroot,
+        "-cmd", "bootstrap", "-solrhome", ExternalPaths.EXAMPLE_HOME};
+    
+    ZkCLI.main(args);
+    
+    assertTrue(zkClient.exists(chroot + ZkController.CONFIGS_ZKNODE
+        + "/collection1", true));
   }
 
   @Test