SOLR-3460: Add cloud-scripts directory and a zkcli.sh|bat tool for easy scripting and interaction with ZooKeeper.

SOLR-3609: Change Solr's expanded webapp directory to be at a consistent path called solr-webapp rather than a temporary directory.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1360416 13f79535-47bb-0310-9956-ffa450edef68

dev-tools/eclipse/dot.classpath

@@ -103,6 +103,7 @@
 	<classpathentry kind="lib" path="lucene/benchmark/lib/commons-compress-1.2.jar"/>
 	<classpathentry kind="lib" path="lucene/benchmark/lib/xercesImpl-2.9.1.jar"/>
 	<classpathentry kind="lib" path="solr/lib/commons-fileupload-1.2.1.jar"/>
+	<classpathentry kind="lib" path="solr/lib/commons-cli-1.2.jar"/>
 	<classpathentry kind="lib" path="solr/lib/httpclient-4.1.3.jar"/>
 	<classpathentry kind="lib" path="solr/lib/httpcore-4.1.4.jar"/>
 	<classpathentry kind="lib" path="solr/lib/httpmime-4.1.3.jar"/>

dev-tools/maven/pom.xml.template

@@ -155,6 +155,11 @@
         <artifactId>commons-codec</artifactId>
         <version>1.6</version>
       </dependency>
+      <dependency>
+        <groupId>commons-cli</groupId>
+        <artifactId>commons-cli</artifactId>
+        <version>1.2</version>
+      </dependency>
       <dependency>
         <groupId>commons-digester</groupId>
         <artifactId>commons-digester</artifactId>

dev-tools/maven/solr/core/pom.xml.template

@@ -138,6 +138,10 @@
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
     </dependency>
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+    </dependency>
     <dependency>
       <groupId>commons-fileupload</groupId>
       <artifactId>commons-fileupload</artifactId>

solr/CHANGES.txt

@@ -57,6 +57,9 @@ New Features
 * SOLR-2616: Include a sample java util logging configuration file. 
   (David Smiley, Mark Miller)
  
+* SOLR-3460: Add cloud-scripts directory and a zkcli.sh|bat tool for easy scripting
+  and interaction with ZooKeeper. (Mark Miller)
+
 
 Bug Fixes
 
@@ -88,6 +91,8 @@ Other Changes
 * SOLR-3599: Add zkClientTimeout to solr.xml so that it's obvious how to change it and so 
   that you can change it with a system property. (Mark Miller)
 
+* SOLR-3609: Change Solr's expanded webapp directory to be at a consistent path called
+  solr-webapp rather than a temporary directory. (Mark Miller)
 
 ==================  4.0.0-ALPHA ==================
 More information about this release, including any errata related to the 

solr/cloud-dev/cli-test-solrcloud-start.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+cd ..
+
+rm -r -f example2
+rm -r -f example3
+rm -r -f example4
+rm -r -f example5
+rm -r -f example6
+
+rm -r -f dist
+rm -r -f build
+rm -r -f example/solr/zoo_data
+rm -r -f example/solr/data
+rm -f example/example.log
+
+ant example dist
+
+cp -r -f example example2
+cp -r -f example example3
+cp -r -f example example4
+cp -r -f example example5
+cp -r -f example example6
+
+# first try uploading a conf dir
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd upconfig -zkhost 127.0.0.1:9983 -confdir example/solr/collection1/conf -confname conf1 -solrhome example/solr -runzk 8983
+
+# upload a second conf set so we avoid single conf auto linking
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd upconfig -zkhost 127.0.0.1:9983 -confdir example/solr/collection1/conf -confname conf2 -solrhome example/solr -runzk 8983
+
+# now try linking a collection to a conf set
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd linkconfig -zkhost 127.0.0.1:9983 -collection collection1 -confname conf1 -solrhome example/solr -runzk 8983
+
+
+cd example
+java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example.log 2>&1 &
+
+cd ../example2
+java -Djetty.port=7574 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6574 -DSTOP.KEY=key -jar start.jar 1>example2.log 2>&1 &
+
+cd ../example3
+java -Djetty.port=7575 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6575 -DSTOP.KEY=key -jar start.jar 1>example3.log 2>&1 &
+
+cd ../example4
+java -Djetty.port=7576 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6576 -DSTOP.KEY=key -jar start.jar 1>example4.log 2>&1 &
+
+cd ../example5
+java -Djetty.port=7577 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6577 -DSTOP.KEY=key -jar start.jar 1>example5.log 2>&1 &
+
+cd ../example6
+java -Djetty.port=7578 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6578 -DSTOP.KEY=key -jar start.jar 1>example6.log 2>&1 &

solr/cloud-dev/solrcloud-multi-start.sh

@@ -24,7 +24,7 @@ cp -r -f example example4
 cp -r -f example example5
 cp -r -f example example6
 
-java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983 example/multicore 8983
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd upconf -zkhost 127.0.0.1:9983 -solrhome example/multicore -runzk 8983
 
 cd example
 java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dsolr.solr.home=multicore -jar start.jar 1>example.log 2>&1 &

solr/cloud-dev/solrcloud-start.sh

@@ -22,7 +22,7 @@ cp -r -f example example4
 cp -r -f example example5
 cp -r -f example example6
 
-java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983 example/solr 8983
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd upconf -zkhost 127.0.0.1:9983 -solrhome example/solr -runzk 8983
 
 cd example
 java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example.log 2>&1 &

solr/core/ivy.xml

@@ -22,6 +22,7 @@
     <dependencies>
       <dependency org="commons-codec" name="commons-codec" rev="1.6" transitive="false"/>
       <dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" transitive="false"/>
+      <dependency org="commons-cli" name="commons-cli" rev="1.2" transitive="false"/>
       <dependency org="org.apache.httpcomponents" name="httpcore" rev="4.1.4" transitive="false"/>
       <dependency org="org.apache.httpcomponents" name="httpclient" rev="4.1.3" transitive="false"/>
       <dependency org="org.apache.httpcomponents" name="httpmime" rev="4.1.3" transitive="false"/>

solr/core/src/java/org/apache/solr/cloud/ZkCLI.java

@@ -0,0 +1,244 @@
+package org.apache.solr.cloud;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.solr.common.cloud.OnReconnect;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.core.Config;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.zookeeper.KeeperException;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class ZkCLI {
+  
+  private static final String MAKEPATH = "makepath";
+  private static final String DOWNCONFIG = "downconfig";
+  private static final String ZK_CLI_NAME = "ZkCLI";
+  private static final String HELP = "help";
+  private static final String LINKCONFIG = "linkconfig";
+  private static final String CONFDIR = "confdir";
+  private static final String CONFNAME = "confname";
+  private static final String REVERSE = "reverse";
+  private static final String ZKHOST = "zkhost";
+  private static final String RUNZK = "runzk";
+  private static final String SOLRHOME = "solrhome";
+  private static final String BOOTSTRAP = "bootstrap";
+  private static final String SOLR_XML = "solr.xml";
+  private static final String UPCONFIG = "upconfig";
+  private static final String COLLECTION = "collection";
+  private static final String CLEAR = "clear";
+  private static final String CMD = "cmd";
+  
+  /**
+   * Allows you to perform a variety of zookeeper related tasks, such as:
+   * 
+   * Bootstrap the current configs for all collections in solr.xml.
+   * 
+   * Upload a named config set from a given directory.
+   * 
+   * Link a named config set explicity to a collection.
+   * 
+   * Clear ZooKeeper info.
+   * 
+   * If you also pass a solrPort, it will be used to start an embedded zk useful
+   * for single machine, multi node tests.
+   * 
+   * @param args
+   * @throws IOException
+   * @throws TimeoutException
+   * @throws InterruptedException
+   * @throws SAXException
+   * @throws ParserConfigurationException
+   * @throws KeeperException
+   */
+  public static void main(String[] args) throws InterruptedException,
+      TimeoutException, IOException, ParserConfigurationException,
+      SAXException, KeeperException {
+
+    CommandLineParser parser = new PosixParser();
+    Options options = new Options();
+    
+    options.addOption(OptionBuilder
+        .hasArg(true)
+        .withDescription(
+            "cmd to run: " + BOOTSTRAP + ", " + UPCONFIG + ", " + DOWNCONFIG
+                + ", " + LINKCONFIG + ", " + MAKEPATH + ", "+ CLEAR).create(CMD));
+
+    Option zkHostOption = new Option("z", ZKHOST, true,
+        "ZooKeeper host address");
+    options.addOption(zkHostOption);
+    Option solrHomeOption = new Option("s", SOLRHOME, true,
+        "for " + BOOTSTRAP + ", " + RUNZK + ": solrhome location");
+    options.addOption(zkHostOption);
+    options.addOption(solrHomeOption);
+    
+    options.addOption("d", CONFDIR, true,
+        "for " + UPCONFIG + ": a directory of configuration files");
+    options.addOption("n", CONFNAME, true,
+        "for " + UPCONFIG + ", " + LINKCONFIG + ": name of the config set");
+
+    
+    options.addOption("c", COLLECTION, true,
+        "for " + LINKCONFIG + ": name of the collection");
+    
+    options
+        .addOption(
+            "r",
+            RUNZK,
+            true,
+            "run zk internally by passing the solr run port - only for clusters on one machine (tests, dev)");
+    
+    options.addOption("h", HELP, false, "bring up this help page");
+    
+    try {
+      // parse the command line arguments
+      CommandLine line = parser.parse(options, args);
+      
+      if (line.hasOption(HELP) || !line.hasOption(ZKHOST)
+          || !line.hasOption(CMD)) {
+        // automatically generate the help statement
+        HelpFormatter formatter = new HelpFormatter();
+        formatter.printHelp(ZK_CLI_NAME, options);
+        System.out.println("Examples:");
+        System.out.println("zkcli.sh -cmd " + BOOTSTRAP + " -" + SOLRHOME + " /opt/solr");
+        System.out.println("zkcli.sh -cmd " + UPCONFIG + " -" + CONFDIR + " /opt/solr/collection1/conf" + " -" + CONFNAME + " myconf");
+        System.out.println("zkcli.sh -cmd " + DOWNCONFIG + " -" + CONFDIR + " /opt/solr/collection1/conf" + " -" + CONFNAME + " myconf");
+        System.out.println("zkcli.sh -cmd " + LINKCONFIG + " -" + COLLECTION + " collection1" + " -" + CONFNAME + " myconf");
+        System.out.println("zkcli.sh -cmd " + MAKEPATH + " /apache/solr");
+        System.out.println("zkcli.sh -cmd " + CLEAR + " /solr");
+        return;
+      }
+      
+      // start up a tmp zk server first
+      String zkServerAddress = line.getOptionValue(ZKHOST);
+      String solrHome = line.getOptionValue(SOLRHOME);
+      
+      String solrPort = null;
+      if (line.hasOption(RUNZK)) {
+        if (!line.hasOption(SOLRHOME)) {
+          System.out
+              .println("-" + SOLRHOME + " is required for " + RUNZK);
+          System.exit(1);
+        }
+        solrPort = line.getOptionValue(RUNZK);
+      }
+      
+      SolrZkServer zkServer = null;
+      if (solrPort != null) {
+        zkServer = new SolrZkServer("true", null, solrHome + "/zoo_data",
+            solrHome, solrPort);
+        zkServer.parseConfig();
+        zkServer.start();
+      }
+      SolrZkClient zkClient = null;
+      try {
+        zkClient = new SolrZkClient(zkServerAddress, 15000, 5000,
+            new OnReconnect() {
+              @Override
+              public void command() {}
+            });
+        
+        if (line.getOptionValue(CMD).equals(BOOTSTRAP)) {
+          if (!line.hasOption(SOLRHOME)) {
+            System.out.println("-" + SOLRHOME
+                + " is required for " + BOOTSTRAP);
+            System.exit(1);
+          }
+          SolrResourceLoader loader = new SolrResourceLoader(solrHome);
+          solrHome = loader.getInstanceDir();
+          
+          InputSource cfgis = new InputSource(new File(solrHome, SOLR_XML)
+              .toURI().toASCIIString());
+          Config cfg = new Config(loader, null, cfgis, null, false);
+          ZkController.bootstrapConf(zkClient, cfg, solrHome);
+          
+        } else if (line.getOptionValue(CMD).equals(UPCONFIG)) {
+          if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
+            System.out.println("-" + CONFDIR + " and -" + CONFNAME
+                + " are required for " + UPCONFIG);
+            System.exit(1);
+          }
+          String confDir = line.getOptionValue(CONFDIR);
+          String confName = line.getOptionValue(CONFNAME);
+          
+          ZkController.uploadConfigDir(zkClient, new File(confDir), confName);
+        } else if (line.getOptionValue(CMD).equals(DOWNCONFIG)) {
+          if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
+            System.out.println("-" + CONFDIR + " and -" + CONFNAME
+                + " are required for " + DOWNCONFIG);
+            System.exit(1);
+          }
+          String confDir = line.getOptionValue(CONFDIR);
+          String confName = line.getOptionValue(CONFNAME);
+          
+          ZkController.downloadConfigDir(zkClient, confName, new File(confDir));
+        } else if (line.getOptionValue(CMD).equals(LINKCONFIG)) {
+          if (!line.hasOption(COLLECTION) || !line.hasOption(CONFNAME)) {
+            System.out.println("-" + CONFDIR + " and -" + CONFNAME
+                + " are required for " + LINKCONFIG);
+            System.exit(1);
+          }
+          String collection = line.getOptionValue(COLLECTION);
+          String confName = line.getOptionValue(CONFNAME);
+          
+          ZkController.linkConfSet(zkClient, collection, confName);
+        } else if (line.getOptionValue(CMD).equals(CLEAR)) {
+          List arglist = line.getArgList();
+          if (arglist.size() != 1) {
+            System.out.println("-" + CLEAR + " requires one arg - the path to clear");
+            System.exit(1);
+          }
+          zkClient.clean(arglist.get(0).toString());
+        } else if (line.getOptionValue(CMD).equals(MAKEPATH)) {
+          List arglist = line.getArgList();
+          if (arglist.size() != 1) {
+            System.out.println("-" + MAKEPATH + " requires one arg - the path to make");
+            System.exit(1);
+          }
+          zkClient.makePath(arglist.get(0).toString(), true);
+        }
+      } finally {
+        if (solrPort != null) {
+          zkServer.stop();
+        }
+        if (zkClient != null) {
+          zkClient.close();
+        }
+      }
+    } catch (ParseException exp) {
+      System.out.println("Unexpected exception:" + exp.getMessage());
+    }
+    
+  }
+}

solr/core/src/java/org/apache/solr/cloud/ZkController.java

@@ -33,6 +33,7 @@ import java.util.regex.Pattern;
 
 import javax.xml.xpath.XPathConstants;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
 import org.apache.solr.common.SolrException;
@@ -50,8 +51,8 @@ import org.apache.solr.core.Config;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrResourceLoader;
 import org.apache.solr.handler.component.HttpShardHandlerFactory;
+
 import org.apache.solr.handler.component.ShardHandler;
 import org.apache.solr.update.UpdateLog;
 import org.apache.solr.util.DOMUtil;
@@ -63,7 +64,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
 
 /**
  * Handle ZooKeeper interactions.
@@ -121,53 +121,6 @@ public final class ZkController {
   // may accept defaults or use mocks rather than pulling things from a CoreContainer
   private CoreContainer cc;
 
-  /**
-   * Bootstraps the current configs for all collections in solr.xml.
-   * Takes two params - the zkhost to connect to and the solrhome location
-   * to find solr.xml.
-   *
-   * If you also pass a solrPort, it will be used to start
-   * an embedded zk useful for single machine, multi node tests.
-   * 
-   * @param args
-   * @throws Exception
-   */
-  public static void main(String[] args) throws Exception {
-    // start up a tmp zk server first
-    String zkServerAddress = args[0];
-    
-    String solrHome = args[1];
-   
-    String solrPort = null;
-    if (args.length > 2) {
-      solrPort = args[2];
-    }
-    
-
-    SolrZkServer zkServer = null;
-    if (solrPort != null) {
-      zkServer = new SolrZkServer("true", null, solrHome + "/zoo_data", solrHome, solrPort);
-      zkServer.parseConfig();
-      zkServer.start();
-    }
-    
-    SolrZkClient zkClient = new SolrZkClient(zkServerAddress, 15000, 5000,
-        new OnReconnect() {
-          @Override
-          public void command() {
-          }});
-    
-    SolrResourceLoader loader = new SolrResourceLoader(solrHome);
-    solrHome = loader.getInstanceDir();
-    
-    InputSource cfgis = new InputSource(new File(solrHome, "solr.xml").toURI().toASCIIString());
-    Config cfg = new Config(loader, null, cfgis , null, false);
-    bootstrapConf(zkClient, cfg, solrHome);
-    if (solrPort != null) {
-      zkServer.stop();
-    }
-  }
-
   /**
    * @param cc if null, recovery will not be enabled
    * @param zkServerAddress
@@ -899,8 +852,6 @@ public final class ZkController {
           ZkNodeProps zkProps = new ZkNodeProps(collectionProps);
           zkClient.makePath(collectionPath, ZkStateReader.toJSON(zkProps), CreateMode.PERSISTENT, null, true);
 
-          // ping that there is a new collection
-          zkClient.setData(ZkStateReader.COLLECTIONS_ZKNODE, (byte[])null, true);
         } catch (KeeperException e) {
           // its okay if the node already exists
           if (e.code() != KeeperException.Code.NODEEXISTS) {
@@ -1008,6 +959,24 @@ public final class ZkController {
     }
   }
   
+  public static void downloadFromZK(SolrZkClient zkClient, String zkPath,
+      File dir) throws IOException, KeeperException, InterruptedException {
+    List<String> files = zkClient.getChildren(zkPath, null, true);
+    
+    for (String file : files) {
+      List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true);
+      if (children.size() == 0) {
+        byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);
+        dir.mkdirs(); 
+        log.info("Write file " + new File(dir, file));
+        FileUtils.writeStringToFile(new File(dir, file), new String(data, "UTF-8"), "UTF-8");
+      } else {
+        downloadFromZK(zkClient, zkPath + "/" + file, new File(dir, file));
+      }
+    }
+  }
+  
+  
   private String getCoreNodeName(CoreDescriptor descriptor){
     return getNodeName() + "_"
         + descriptor.getName();
@@ -1017,6 +986,10 @@ public final class ZkController {
     uploadToZK(zkClient, dir, ZkController.CONFIGS_ZKNODE + "/" + configName);
   }
   
+  public static void downloadConfigDir(SolrZkClient zkClient, String configName, File dir) throws IOException, KeeperException, InterruptedException {
+    downloadFromZK(zkClient, ZkController.CONFIGS_ZKNODE + "/" + configName, dir);
+  }
+
   public void preRegister(CoreDescriptor cd) throws KeeperException, InterruptedException {
     // before becoming available, make sure we are not live and active
     // this also gets us our assigned shard id if it was not specified
@@ -1100,6 +1073,50 @@ public final class ZkController {
     return leaderProps;
   }
   
+  public static void linkConfSet(SolrZkClient zkClient, String collection, String confSetName) throws KeeperException, InterruptedException {
+    String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
+    if (log.isInfoEnabled()) {
+      log.info("Load collection config from:" + path);
+    }
+    byte[] data;
+    try {
+      data = zkClient.getData(path, null, null, true);
+    } catch (NoNodeException e) {
+      // if there is no node, we will try and create it
+      // first try to make in case we are pre configuring
+      ZkNodeProps props = new ZkNodeProps(CONFIGNAME_PROP, confSetName);
+      try {
+
+        zkClient.makePath(path, ZkStateReader.toJSON(props),
+            CreateMode.PERSISTENT, null, true);
+      } catch (KeeperException e2) {
+        // its okay if the node already exists
+        if (e2.code() != KeeperException.Code.NODEEXISTS) {
+          throw e;
+        }
+        // if we fail creating, setdata
+        // TODO: we should consider using version
+        zkClient.setData(path, ZkStateReader.toJSON(props), true);
+      }
+      return;
+    }
+    // we found existing data, let's update it
+    ZkNodeProps props = null;
+    if(data != null) {
+      props = ZkNodeProps.load(data);
+      Map<String,String> newProps = new HashMap<String,String>();
+      newProps.putAll(props.getProperties());
+      newProps.put(CONFIGNAME_PROP, confSetName);
+      props = new ZkNodeProps(newProps);
+    } else {
+      props = new ZkNodeProps(CONFIGNAME_PROP, confSetName);
+    }
+    
+    // TODO: we should consider using version
+    zkClient.setData(path, ZkStateReader.toJSON(props), true);
+
+  }
+  
   /**
    * If in SolrCloud mode, upload config sets for each SolrCore in solr.xml.
    * 
@@ -1114,18 +1131,20 @@ public final class ZkController {
 
     for (int i=0; i<nodes.getLength(); i++) {
       Node node = nodes.item(i);
-      String rawName = DOMUtil.getAttr(node, "name", null);
+      String rawName = DOMUtil.substituteProperty(DOMUtil.getAttr(node, "name", null), new Properties());
+
       String instanceDir = DOMUtil.getAttr(node, "instanceDir", null);
       File idir = new File(instanceDir);
       if (!idir.isAbsolute()) {
         idir = new File(solrHome, instanceDir);
       }
-      String confName = DOMUtil.getAttr(node, "collection", null);
+      String confName = DOMUtil.substituteProperty(DOMUtil.getAttr(node, "collection", null), new Properties());
       if (confName == null) {
         confName = rawName;
       }
-
+      File udir = new File(idir, "conf");
-      ZkController.uploadConfigDir(zkClient, new File(idir, "conf"), confName);
+      SolrException.log(log, "Uploading directory " + udir + " with name " + confName + " for SolrCore " + rawName);
+      ZkController.uploadConfigDir(zkClient, udir, confName);
     }
   }
 

solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java

@@ -0,0 +1,172 @@
+package org.apache.solr.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Slow
+// TODO: This test would be a lot faster if it used a solrhome with fewer config
+// files - there are a lot of them to upload
+public class ZkCLITest extends SolrTestCaseJ4 {
+  protected static Logger log = LoggerFactory
+      .getLogger(AbstractZkTestCase.class);
+  
+  private static final boolean VERBOSE = false;
+  
+  protected ZkTestServer zkServer;
+  
+  protected String zkDir;
+
+  private SolrZkClient zkClient;
+  
+  
+  @BeforeClass
+  public static void beforeClass() {
+    System.setProperty("solrcloud.skip.autorecovery", "true");
+  }
+  
+  @AfterClass
+  public static void afterClass() throws InterruptedException {
+    System.clearProperty("solrcloud.skip.autorecovery");
+  }
+  
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    log.info("####SETUP_START " + getTestName());
+    createTempDir();
+    
+    zkDir = dataDir.getAbsolutePath() + File.separator
+        + "zookeeper/server1/data";
+    zkServer = new ZkTestServer(zkDir);
+    zkServer.run();
+    System.setProperty("zkHost", zkServer.getZkAddress());
+    AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(),
+        zkServer.getZkAddress(), "solrconfig.xml", "schema.xml");
+
+    
+    zkClient = new SolrZkClient(zkServer.getZkAddress(),
+        AbstractZkTestCase.TIMEOUT);
+    
+    log.info("####SETUP_END " + getTestName());
+    
+  }
+  
+  @Test
+  public void testBootstrap() throws Exception {
+    // test bootstrap_conf
+    String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "bootstrap", "-solrhome", TEST_HOME()};
+    ZkCLI.main(args);
+
+
+    assertTrue(zkClient.exists(ZkStateReader.COLLECTIONS_ZKNODE + "/collection1", true));
+    
+    assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/collection1", true));
+    
+  }
+  
+  @Test
+  public void testMakePath() throws Exception {
+    // test bootstrap_conf
+    String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "makepath", "/path/mynewpath"};
+    ZkCLI.main(args);
+
+
+    assertTrue(zkClient.exists("/path/mynewpath", true));
+  }
+  
+  @Test
+  public void testUpConfigLinkConfigClearZk() throws Exception {
+    // test upconfig
+    String confsetname = "confsetone";
+    String[] args = new String[] {
+        "-zkhost",
+        zkServer.getZkAddress(),
+        "-cmd",
+        "upconfig",
+        "-confdir",
+        TEST_HOME() + File.separator + "collection1"
+            + File.separator + "conf", "-confname", confsetname};
+    ZkCLI.main(args);
+    
+    assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/" + confsetname, true));
+
+    // print help
+    ZkCLI.main(new String[0]);
+    
+    // test linkconfig
+    args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "linkconfig", "-collection", "collection1", "-confname", confsetname};
+    ZkCLI.main(args);
+    
+    ZkNodeProps collectionProps = ZkNodeProps.load(zkClient.getData(ZkStateReader.COLLECTIONS_ZKNODE + "/collection1", null, null, true));
+    assertTrue(collectionProps.containsKey("configName"));
+    assertEquals(confsetname, collectionProps.get("configName"));
+    
+    // test down config
+    File confDir = new File(TEMP_DIR,
+        "solrtest-confdropspot-" + this.getClass().getName() + "-" + System.currentTimeMillis());
+    
+    assertFalse(confDir.exists());
+    
+    args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "downconfig", "-confdir", confDir.getAbsolutePath(), "-confname", confsetname};
+    ZkCLI.main(args);
+    
+    File[] files = confDir.listFiles();
+    List<String> zkFiles = zkClient.getChildren(ZkController.CONFIGS_ZKNODE + "/" + confsetname, null, true);
+    assertEquals(files.length, zkFiles.size());
+    
+    // test reset zk
+    args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "clear", "/"};
+    ZkCLI.main(args);
+
+    assertEquals(0, zkClient.getChildren("/", null, true).size());
+  }
+  
+  @Override
+  public void tearDown() throws Exception {
+    if (VERBOSE) {
+      printLayout(zkServer.getZkHost());
+    }
+    zkClient.close();
+    zkServer.shutdown();
+    super.tearDown();
+  }
+  
+  private void printLayout(String zkHost) throws Exception {
+    SolrZkClient zkClient = new SolrZkClient(zkHost, AbstractZkTestCase.TIMEOUT);
+    zkClient.printLayoutToStdOut();
+    zkClient.close();
+  }
+}

solr/example/cloud-scripts/zkcli.bat

@@ -0,0 +1,12 @@
+REM You can override pass the following parameters to this script:
+REM 
+
+set JVM=java
+
+REM  Find location of this script
+
+set SDIR=%~dp0
+if "%SDIR:~-1%"=="\" set SDIR=%SDIR:~0,-1%
+
+     
+"%JVM%" -classpath "%SDIR%\..\solr-webapp\webapp\WEB-INF\lib\*" org.apache.solr.cloud.ZkCLI %*

solr/example/cloud-scripts/zkcli.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# You can override pass the following parameters to this script:
+# 
+
+JVM="java"
+
+# Find location of this script
+
+sdir="`dirname \"$0\"`"
+
+
+$JVM  -classpath "$sdir/../solr-webapp/webapp/WEB-INF/lib/*" org.apache.solr.cloud.ZkCLI ${1+"$@"}
+

solr/example/contexts/solr.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure.dtd">
+<Configure class="org.eclipse.jetty.webapp.WebAppContext">
+  <Set name="contextPath">/solr</Set>
+  <Set name="war"><SystemProperty name="jetty.home"/>/webapps/solr.war</Set>
+  <Set name="defaultsDescriptor"><SystemProperty name="jetty.home"/>/etc/webdefault.xml</Set>
+  <Set name="tempDirectory"><Property name="jetty.home" default="."/>/solr-webapp</Set>
+</Configure>

solr/example/etc/jetty.xml

@@ -134,17 +134,14 @@
     </Call>
     
     <Ref id="DeploymentManager">
-          <Call id="webappprovider" name="addAppProvider">
+      <Call name="addAppProvider">
-            <Arg>
+        <Arg>
-              <New class="org.eclipse.jetty.deploy.providers.WebAppProvider">
+          <New class="org.eclipse.jetty.deploy.providers.ContextProvider">
-                <Set name="monitoredDirName"><Property name="jetty.home" default="." />/webapps</Set>
+            <Set name="monitoredDirName"><SystemProperty name="jetty.home" default="."/>/contexts</Set>
-                <Set name="defaultsDescriptor"><Property name="jetty.home" default="."/>/etc/webdefault.xml</Set>
+            <Set name="scanInterval">0</Set>
-                <Set name="scanInterval">0</Set>
+          </New>
-                <Set name="contextXmlDir"><Property name="jetty.home" default="." />/contexts</Set>
+        </Arg>
-		<Set name="extractWars">true</Set>
+      </Call>
-              </New>
-            </Arg>
-          </Call>
     </Ref>
 
 </Configure>

solr/example/work/.gitignore

@@ -1 +0,0 @@
-*

solr/lib/commons-cli-LICENSE-ASL.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

solr/lib/commons-cli-NOTICE.txt

@@ -0,0 +1,5 @@
+Apache Commons CLI
+Copyright 2001-2009 The Apache Software Foundation
+
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).

solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java

@@ -689,10 +689,16 @@ public class SolrZkClient {
       return;
     }
     for (String string : children) {
-      clean(path + "/" + string);
+      if (path.equals("/")) {
+        clean(path + string);
+      } else {
+        clean(path + "/" + string);
+      }
     }
     try {
-      delete(path, -1, true);
+      if (!path.equals("/")) {
+        delete(path, -1, true);
+      }
     } catch (NoNodeException r) {
       return;
     }