SOLR-5156 Provide a way to move the contents of a file to ZooKeeper with ZkCLI

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1514776 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -92,6 +92,7 @@ New Features
 * SOLR-5006: CREATESHARD command for 'implicit' shards (Noble Paul)
 * SOLR-5017: Allow sharding based on the value of a field (Noble Paul)
 * SOLR-4222: create custom sharded collection via collections API (Noble Paul)
+* SOLR-5156: Enhance ZkCLI to allow uploading of arbitrary files to ZK.
 
 Bug Fixes
 ----------------------
@@ -130,8 +131,6 @@ Bug Fixes
   of divide by zero, and makes estimated hit counts meaningful in non-optimized
   indexes.  (hossman)
   
-* SOLR-5164: Can not create a collection via collections API (cloud mode) (Erick Erickson)
-
 Optimizations
 ----------------------
 

solr/core/src/java/org/apache/solr/cloud/ZkCLI.java

@@ -3,6 +3,7 @@ package org.apache.solr.cloud;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
@@ -19,7 +20,9 @@ import org.xml.sax.SAXException;
 
 import javax.xml.parsers.ParserConfigurationException;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
 
@@ -44,6 +47,7 @@ public class ZkCLI {
   
   private static final String MAKEPATH = "makepath";
   private static final String PUT = "put";
+  private static final String PUT_FILE = "putfile";
   private static final String DOWNCONFIG = "downconfig";
   private static final String ZK_CLI_NAME = "ZkCLI";
   private static final String HELP = "help";
@@ -87,7 +91,8 @@ public class ZkCLI {
         .hasArg(true)
         .withDescription(
             "cmd to run: " + BOOTSTRAP + ", " + UPCONFIG + ", " + DOWNCONFIG
-                + ", " + LINKCONFIG + ", " + MAKEPATH + ", "+ PUT + ", "+ LIST + ", " + CLEAR).create(CMD));
+                + ", " + LINKCONFIG + ", " + MAKEPATH + ", " + PUT + ", " + PUT_FILE + ","
+                + LIST + ", " + CLEAR).create(CMD));
 
     Option zkHostOption = new Option("z", ZKHOST, true,
         "ZooKeeper host address");
@@ -131,6 +136,7 @@ public class ZkCLI {
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LINKCONFIG + " -" + COLLECTION + " collection1" + " -" + CONFNAME + " myconf");
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr");
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT + " /solr.conf 'conf data'");
+        System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT_FILE + " /solr.xml /User/myuser/solr/solr.xml");
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
         System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
         return;
@@ -244,6 +250,20 @@ public class ZkCLI {
           }
           zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes("UTF-8"),
                           acl, CreateMode.PERSISTENT, true);
+        } else if (line.getOptionValue(CMD).equals(PUT_FILE)) {
+          List arglist = line.getArgList();
+          if (arglist.size() != 2) {
+            System.out.println("-" + PUT_FILE + " requires two args - the path to create in ZK and the path to the local file");
+            System.exit(1);
+          }
+          InputStream is = new FileInputStream(arglist.get(1).toString());
+          try {
+            zkClient.create(arglist.get(0).toString(), IOUtils.toByteArray(is),
+                ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true);
+          } finally {
+            IOUtils.closeQuietly(is);
+          }
+
         }
       } finally {
         if (solrPort != null) {

solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java

@@ -18,10 +18,14 @@ package org.apache.solr.cloud;
  */
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
 import java.util.Collection;
 import java.util.List;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.filefilter.RegexFileFilter;
 import org.apache.commons.io.filefilter.TrueFileFilter;
 import org.apache.solr.SolrTestCaseJ4;
@@ -51,6 +55,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
 
   private SolrZkClient zkClient;
 
+  protected static final String SOLR_HOME = SolrTestCaseJ4.TEST_HOME();
   
   @BeforeClass
   public static void beforeClass() {
@@ -141,7 +146,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
 
   @Test
   public void testPut() throws Exception {
-    // test bootstrap_conf
+    // test put
     String data = "my data";
     String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
         "put", "/data.txt", data};
@@ -152,6 +157,40 @@ public class ZkCLITest extends SolrTestCaseJ4 {
     assertArrayEquals(zkClient.getData("/data.txt", null, null, true), data.getBytes("UTF-8"));
   }
 
+  @Test
+  public void testPutFile() throws Exception {
+    // test put file
+    String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "putfile", "/solr.xml", SOLR_HOME + File.separator + "solr-stress-new.xml"};
+    ZkCLI.main(args);
+
+    String fromZk = new String(zkClient.getData("/solr.xml", null, null, true), "UTF-8");
+    File locFile = new File(SOLR_HOME + File.separator + "solr-stress-new.xml");
+    InputStream is = new FileInputStream(locFile);
+    String fromLoc;
+    try {
+      fromLoc = new String(IOUtils.toByteArray(is), "UTF-8");
+    } finally {
+      IOUtils.closeQuietly(is);
+    }
+    assertEquals("Should get back what we put in ZK", fromZk, fromLoc);
+  }
+
+  @Test
+  public void testPutFileNotExists() throws Exception {
+    // test put file
+    String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
+        "putfile", "/solr.xml", SOLR_HOME + File.separator + "not-there.xml"};
+    try {
+      ZkCLI.main(args);
+      fail("Should have had a file not found exception");
+    } catch (FileNotFoundException fne) {
+      String msg = fne.getMessage();
+      assertTrue("Didn't find expected error message containing 'not-there.xml' in " + msg,
+          msg.indexOf("not-there.xml") != -1);
+    }
+  }
+
   @Test
   public void testList() throws Exception {
     zkClient.makePath("/test", true);