From b8097e9e5307660b1edbf22c38455277ac3608db Mon Sep 17 00:00:00 2001 From: Erick Erickson Date: Fri, 16 Aug 2013 16:30:17 +0000 Subject: [PATCH] SOLR-5156 Provide a way to move the contents of a file to ZooKeeper with ZkCLI git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1514776 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 5 +-- .../src/java/org/apache/solr/cloud/ZkCLI.java | 22 ++++++++- .../test/org/apache/solr/cloud/ZkCLITest.java | 45 +++++++++++++++++-- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 31503e157c3..8b5edf33c1d 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -91,7 +91,8 @@ New Features * SOLR-4808: Persist and use router,replicationFactor and maxShardsPerNode at Collection and Shard level (Noble Paul, Shalin Mangar) * SOLR-5006: CREATESHARD command for 'implicit' shards (Noble Paul) * SOLR-5017: Allow sharding based on the value of a field (Noble Paul) -* SOLR-4222:create custom sharded collection via collections API (Noble Paul) +* SOLR-4222: create custom sharded collection via collections API (Noble Paul) +* SOLR-5156: Enhance ZkCLI to allow uploading of arbitrary files to ZK. Bug Fixes ---------------------- @@ -130,8 +131,6 @@ Bug Fixes of divide by zero, and makes estimated hit counts meaningful in non-optimized indexes. (hossman) -* SOLR-5164: Can not create a collection via collections API (cloud mode) (Erick Erickson) - Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java index 1dc042a19f6..0222102f3e2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java @@ -3,6 +3,7 @@ package org.apache.solr.cloud; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.io.IOUtils; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; @@ -19,7 +20,9 @@ import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.concurrent.TimeoutException; @@ -44,6 +47,7 @@ public class ZkCLI { private static final String MAKEPATH = "makepath"; private static final String PUT = "put"; + private static final String PUT_FILE = "putfile"; private static final String DOWNCONFIG = "downconfig"; private static final String ZK_CLI_NAME = "ZkCLI"; private static final String HELP = "help"; @@ -87,7 +91,8 @@ public class ZkCLI { .hasArg(true) .withDescription( "cmd to run: " + BOOTSTRAP + ", " + UPCONFIG + ", " + DOWNCONFIG - + ", " + LINKCONFIG + ", " + MAKEPATH + ", "+ PUT + ", "+ LIST + ", " + CLEAR).create(CMD)); + + ", " + LINKCONFIG + ", " + MAKEPATH + ", " + PUT + ", " + PUT_FILE + "," + + LIST + ", " + CLEAR).create(CMD)); Option zkHostOption = new Option("z", ZKHOST, true, "ZooKeeper host address"); @@ -131,6 +136,7 @@ public class ZkCLI { System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LINKCONFIG + " -" + COLLECTION + " collection1" + " -" + CONFNAME + " myconf"); System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr"); System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT + " /solr.conf 'conf data'"); + System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT_FILE + " /solr.xml /User/myuser/solr/solr.xml"); System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr"); System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST); return; @@ -244,6 +250,20 @@ public class ZkCLI { } zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes("UTF-8"), acl, CreateMode.PERSISTENT, true); + } else if (line.getOptionValue(CMD).equals(PUT_FILE)) { + List arglist = line.getArgList(); + if (arglist.size() != 2) { + System.out.println("-" + PUT_FILE + " requires two args - the path to create in ZK and the path to the local file"); + System.exit(1); + } + InputStream is = new FileInputStream(arglist.get(1).toString()); + try { + zkClient.create(arglist.get(0).toString(), IOUtils.toByteArray(is), + ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true); + } finally { + IOUtils.closeQuietly(is); + } + } } finally { if (solrPort != null) { diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java index d51e6ac6e1a..79711ec8044 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java @@ -18,10 +18,14 @@ package org.apache.solr.cloud; */ import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; import java.util.Collection; import java.util.List; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.io.filefilter.RegexFileFilter; import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.solr.SolrTestCaseJ4; @@ -50,7 +54,8 @@ public class ZkCLITest extends SolrTestCaseJ4 { private String solrHome; private SolrZkClient zkClient; - + + protected static final String SOLR_HOME = SolrTestCaseJ4.TEST_HOME(); @BeforeClass public static void beforeClass() { @@ -141,7 +146,7 @@ public class ZkCLITest extends SolrTestCaseJ4 { @Test public void testPut() throws Exception { - // test bootstrap_conf + // test put String data = "my data"; String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "put", "/data.txt", data}; @@ -151,7 +156,41 @@ public class ZkCLITest extends SolrTestCaseJ4 { assertArrayEquals(zkClient.getData("/data.txt", null, null, true), data.getBytes("UTF-8")); } - + + @Test + public void testPutFile() throws Exception { + // test put file + String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", + "putfile", "/solr.xml", SOLR_HOME + File.separator + "solr-stress-new.xml"}; + ZkCLI.main(args); + + String fromZk = new String(zkClient.getData("/solr.xml", null, null, true), "UTF-8"); + File locFile = new File(SOLR_HOME + File.separator + "solr-stress-new.xml"); + InputStream is = new FileInputStream(locFile); + String fromLoc; + try { + fromLoc = new String(IOUtils.toByteArray(is), "UTF-8"); + } finally { + IOUtils.closeQuietly(is); + } + assertEquals("Should get back what we put in ZK", fromZk, fromLoc); + } + + @Test + public void testPutFileNotExists() throws Exception { + // test put file + String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", + "putfile", "/solr.xml", SOLR_HOME + File.separator + "not-there.xml"}; + try { + ZkCLI.main(args); + fail("Should have had a file not found exception"); + } catch (FileNotFoundException fne) { + String msg = fne.getMessage(); + assertTrue("Didn't find expected error message containing 'not-there.xml' in " + msg, + msg.indexOf("not-there.xml") != -1); + } + } + @Test public void testList() throws Exception { zkClient.makePath("/test", true);