From 99550b5c8abbe1886babe26c0e80f74e58864e5d Mon Sep 17 00:00:00 2001 From: Erick Erickson Date: Thu, 10 Dec 2015 15:32:33 +0000 Subject: [PATCH] SOLR-8378: Add upconfig and downconfig commands to the bin/solr script git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1719099 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 + solr/bin/solr | 147 ++++++++++-- solr/bin/solr.cmd | 142 +++++++++-- .../java/org/apache/solr/util/SolrCLI.java | 225 +++++++++++++++--- 4 files changed, 444 insertions(+), 73 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 725ce527c8d..de6e3107144 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -208,6 +208,9 @@ New Features * SOLR-7928: Improve CheckIndex to work against HdfsDirectory (Mike Drob, Gregory Chanan) +* SOLR-8378: Add upconfig and downconfig commands to the bin/solr script + (Erick Erickson) + Bug Fixes ---------------------- diff --git a/solr/bin/solr b/solr/bin/solr index 75eab7ada59..175f420d57c 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -188,13 +188,13 @@ function print_usage() { if [ -z "$CMD" ]; then echo "" echo "Usage: solr COMMAND OPTIONS" - echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version" + echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk" echo "" echo " Standalone server example (start Solr running in the background on port 8984):" echo "" echo " ./solr start -p 8984" echo "" - echo " SolrCloud example (start Solr running in SolrCloud mode using localhost:2181 to connect to ZooKeeper, with 1g max heap size and remote Java debug options enabled):" + echo " SolrCloud example (start Solr running in SolrCloud mode using localhost:2181 to connect to Zookeeper, with 1g max heap size and remote Java debug options enabled):" echo "" echo " ./solr start -c -m 1g -z localhost:2181 -a \"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1044\"" echo "" @@ -208,7 +208,7 @@ function print_usage() { echo " -f Start Solr in foreground; default starts Solr in the background" echo " and sends stdout / stderr to solr-PORT-console.log" echo "" - echo " -c or -cloud Start Solr in SolrCloud mode; if -z not supplied, an embedded ZooKeeper" + echo " -c or -cloud Start Solr in SolrCloud mode; if -z not supplied, an embedded Zookeeper" echo " instance is started on Solr port+1000, such as 9983 if Solr is bound to 8983" echo "" echo " -h Specify the hostname for this Solr instance" @@ -220,8 +220,8 @@ function print_usage() { echo "" echo " -d Specify the Solr server directory; defaults to server" echo "" - echo " -z ZooKeeper connection string; only used when running in SolrCloud mode using -c" - echo " To launch an embedded ZooKeeper instance, don't pass this parameter." + echo " -z Zookeeper connection string; only used when running in SolrCloud mode using -c" + echo " To launch an embedded Zookeeper instance, don't pass this parameter." echo "" echo " -m Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g" echo " results in: -Xms4g -Xmx4g; by default, this script sets the heap size to 512m" @@ -229,7 +229,7 @@ function print_usage() { echo " -s Sets the solr.solr.home system property; Solr will create core directories under" echo " this directory. This allows you to run multiple Solr instances on the same host" echo " while reusing the same server directory set using the -d parameter. If set, the" - echo " specified directory should contain a solr.xml file, unless solr.xml exists in ZooKeeper." + echo " specified directory should contain a solr.xml file, unless solr.xml exists in Zookeeper." echo " This parameter is ignored when running examples (-e), as the solr.solr.home depends" echo " on which example is run. The default value is server/solr." echo "" @@ -266,7 +266,7 @@ function print_usage() { echo "" echo " -c Collection to run healthcheck against." echo "" - echo " -z ZooKeeper connection string; default is localhost:9983" + echo " -z Zookeeper connection string; default is localhost:9983" echo "" elif [ "$CMD" == "status" ]; then echo "" @@ -294,12 +294,12 @@ function print_usage() { echo "" echo " Deletes a core or collection depending on whether Solr is running in standalone (core) or SolrCloud" echo " mode (collection). If you're deleting a collection in SolrCloud mode, the default behavior is to also" - echo " delete the configuration directory from ZooKeeper so long as it is not being used by another collection." + echo " delete the configuration directory from Zookeeper so long as it is not being used by another collection." echo " You can override this behavior by passing -deleteConfig false when running this command." echo "" echo " -c Name of the core / collection to delete" echo "" - echo " -deleteConfig Delete the configuration directory from ZooKeeper; default is true" + echo " -deleteConfig Delete the configuration directory from Zookeeper; default is true" echo "" echo " -p Port of a local Solr instance where you want to delete the core/collection" echo " If not specified, the script will search the local system for a running" @@ -345,14 +345,14 @@ function print_usage() { echo " Alternatively, you can pass the path to your own configuration directory instead of using" echo " one of the built-in configurations, such as: bin/solr create_collection -c mycoll -d /tmp/myconfig" echo "" - echo " By default the script will upload the specified confdir directory into ZooKeeper using the same" + echo " By default the script will upload the specified confdir directory into Zookeeper using the same" echo " name as the collection (-c) option. Alternatively, if you want to reuse an existing directory" - echo " or create a confdir in ZooKeeper that can be shared by multiple collections, use the -n option" + echo " or create a confdir in Zookeeper that can be shared by multiple collections, use the -n option" echo "" - echo " -n Name the configuration directory in ZooKeeper; by default, the configuration" - echo " will be uploaded to ZooKeeper using the collection name (-c), but if you want" + echo " -n Name the configuration directory in Zookeeper; by default, the configuration" + echo " will be uploaded to Zookeeper using the collection name (-c), but if you want" echo " to use an existing directory or override the name of the configuration in" - echo " ZooKeeper, then use the -c option." + echo " Zookeeper, then use the -c option." echo "" echo " -shards <#> Number of shards to split the collection into; default is 1" echo "" @@ -362,6 +362,26 @@ function print_usage() { echo " If not specified, the script will search the local system for a running" echo " Solr instance and will use the port of the first server it finds." echo "" + elif [ "$CMD" == "zk" ]; then + echo "Usage: solr zk [-upconfig|-downconfig] [-d confdir] [-n configName] [-z zkHost]" + echo "" + echo " -upconfig to move a configset from the local machine to Zookeeper." + echo "" + echo " -downconfig to move a configset from Zookeeper to the local machine." + echo "" + echo " -n configName Name of the configset in Zookeeper that will be the destinatino of" + echo " 'upconfig' and the source for 'downconfig'." + echo "" + echo " -d confdir The local directory the configuration will be uploaded from for" + echo " 'upconfig' or downloaded to for 'downconfig'. For 'upconfig', this" + echo " can be one of the example configsets, basic_configs, data_driven_schema_configs or" + echo " sample_techproducts_configs or an arbitrary directory." + echo "" + echo " -z zkHost Zookeeper connection string." + echo "" + echo " NOTE: Solr must have been started least once (or have it running) before using this command." + echo " This initialized Zookeeper for Solr" + echo "" fi } # end print_usage @@ -694,7 +714,7 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM exit 1 fi - # If not defined, use the collection name for the name of the configuration in ZooKeeper + # If not defined, use the collection name for the name of the configuration in Zookeeper if [ -z "$CREATE_CONFNAME" ]; then CREATE_CONFNAME="$CREATE_NAME" fi @@ -784,7 +804,7 @@ if [[ "$SCRIPT_CMD" == "delete" ]]; then exit 1 fi - # If not defined, use the collection name for the name of the configuration in ZooKeeper + # If not defined, use the collection name for the name of the configuration in Zookeeper if [ -z "$DELETE_CONFIG" ]; then DELETE_CONFIG=true fi @@ -810,6 +830,97 @@ if [[ "$SCRIPT_CMD" == "delete" ]]; then exit $? fi +# Upload or download a configset to Zookeeper +if [[ "$SCRIPT_CMD" == "zk" ]]; then + + if [ $# -gt 0 ]; then + while true; do + case "$1" in + -z|-zkhost) + if [[ -z "$2" || "${2:0:1}" == "-" ]]; then + print_usage "$SCRIPT_CMD" "ZooKeepeer connection string is required when using the $1 option!" + exit 1 + fi + ZK_HOST="$2" + shift 2 + ;; + -n|-confname) + if [[ -z "$2" || "${2:0:1}" == "-" ]]; then + print_usage "$SCRIPT_CMD" "Configuration name is required when using the $1 option!" + exit 1 + fi + CONFIGSET_CONFNAME="$2" + shift 2 + ;; + -d|-confdir) + if [[ -z "$2" || "${2:0:1}" == "-" ]]; then + print_usage "$SCRIPT_CMD" "Configuration directory is required when using the $1 option!" + exit 1 + fi + CONFIGSET_CONFDIR="$2" + shift 2 + ;; + -upconfig) + ZK_OP="upconfig" + shift 1 + ;; + -downconfig) + ZK_OP="downconfig" + shift 1 + ;; + -help|-usage|-h) + print_usage "$SCRIPT_CMD" + exit 0 + ;; + --) + shift + break + ;; + *) + if [ "$1" != "" ]; then + print_usage "$SCRIPT_CMD" "Unrecognized or misplaced argument: $1!" + exit 1 + else + break # out-of-args, stop looping + fi + ;; + esac + done + fi + + if [ -z "$ZK_OP" ]; then + echo "Zookeeper operation (one of '-upconfig' or '-downconfig') is required!" + print_usage "$SCRIPT_CMD" + exit 1 + fi + + if [ -z "$ZK_HOST" ]; then + echo "Zookeeper address (-z) argument is required!" + print_usage "$SCRIPT_CMD" + exit 1 + fi + + if [ -z "$CONFIGSET_CONFDIR" ]; then + echo "Local directory of the configset (-d) argument is required!" + print_usage "$SCRIPT_CMD" + exit 1 + fi + + if [ -z "$CONFIGSET_CONFNAME" ]; then + echo "Configset name on Zookeeper (-n) argument is required!" + print_usage "$SCRIPT_CMD" + exit 1 + fi + + if [ "$ZK_OP" == "upconfig" ]; then + run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" -configsetsDir "$SOLR_TIP/server/solr/configsets" + else + run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" + fi + + exit $? +fi + # verify the command given is supported if [ "$SCRIPT_CMD" != "stop" ] && [ "$SCRIPT_CMD" != "start" ] && [ "$SCRIPT_CMD" != "restart" ] && [ "$SCRIPT_CMD" != "status" ]; then print_usage "" "$SCRIPT_CMD is not a valid command!" @@ -900,7 +1011,7 @@ if [ $# -gt 0 ]; then ;; -z|-zkhost) if [[ -z "$2" || "${2:0:1}" == "-" ]]; then - print_usage "$SCRIPT_CMD" "ZooKeeper connection string is required when using the $1 option!" + print_usage "$SCRIPT_CMD" "Zookeeper connection string is required when using the $1 option!" exit 1 fi ZK_HOST="$2" @@ -1174,7 +1285,7 @@ if [ "$SOLR_MODE" == 'solrcloud' ]; then CLOUD_MODE_OPTS+=("-DzkHost=$ZK_HOST") else if $verbose ; then - echo "Configuring SolrCloud to launch an embedded ZooKeeper using -DzkRun" + echo "Configuring SolrCloud to launch an embedded Zookeeper using -DzkRun" fi CLOUD_MODE_OPTS+=('-DzkRun') diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 71273096440..6e0155126e4 100644 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -134,6 +134,12 @@ IF "%1"=="delete" ( SHIFT goto parse_delete_args ) +IF "%1"=="zk" ( + set SCRIPT_CMD=zk + SHIFT + goto parse_zk_args +) + goto parse_args :usage @@ -152,18 +158,19 @@ IF "%SCRIPT_CMD%"=="create" goto create_usage IF "%SCRIPT_CMD%"=="create_core" goto create_core_usage IF "%SCRIPT_CMD%"=="create_collection" goto create_collection_usage IF "%SCRIPT_CMD%"=="delete" goto delete_usage +IF "%SCRIPT_CMD%"=="zk" goto zk_usage goto done :script_usage @echo. @echo Usage: solr COMMAND OPTIONS -@echo where COMMAND is one of: start, stop, restart, healthcheck, create, create_core, create_collection, delete, version +@echo where COMMAND is one of: start, stop, restart, healthcheck, create, create_core, create_collection, delete, version, upconfig, downconfig @echo. @echo Standalone server example (start Solr running in the background on port 8984): @echo. @echo solr start -p 8984 @echo. -@echo SolrCloud example (start Solr running in SolrCloud mode using localhost:2181 to connect to ZooKeeper, with 1g max heap size and remote Java debug options enabled): +@echo SolrCloud example (start Solr running in SolrCloud mode using localhost:2181 to connect to Zookeeper, with 1g max heap size and remote Java debug options enabled): @echo. @echo solr start -c -m 1g -z localhost:2181 -a "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1044" @echo. @@ -179,7 +186,7 @@ goto done @echo -f Start Solr in foreground; default starts Solr in the background @echo and sends stdout / stderr to solr-PORT-console.log @echo. -@echo -c or -cloud Start Solr in SolrCloud mode; if -z not supplied, an embedded ZooKeeper +@echo -c or -cloud Start Solr in SolrCloud mode; if -z not supplied, an embedded Zookeeper @echo instance is started on Solr port+1000, such as 9983 if Solr is bound to 8983 @echo. @echo -h host Specify the hostname for this Solr instance @@ -188,8 +195,8 @@ goto done @echo. @echo -d dir Specify the Solr server directory; defaults to example @echo. -@echo -z zkHost ZooKeeper connection string; only used when running in SolrCloud mode using -c -@echo To launch an embedded ZooKeeper instance, don't pass this parameter. +@echo -z zkHost Zookeeper connection string; only used when running in SolrCloud mode using -c +@echo To launch an embedded Zookeeper instance, don't pass this parameter. @echo. @echo -m memory Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g @echo results in: -Xms4g -Xmx4g; by default, this script sets the heap size to 512m @@ -197,7 +204,7 @@ goto done @echo -s dir Sets the solr.solr.home system property; Solr will create core directories under @echo this directory. This allows you to run multiple Solr instances on the same host @echo while reusing the same server directory set using the -d parameter. If set, the -@echo specified directory should contain a solr.xml file, unless solr.xml exists in ZooKeeper. +@echo specified directory should contain a solr.xml file, unless solr.xml exists in Zookeeper. @echo This parameter is ignored when running examples (-e), as the solr.solr.home depends @echo on which example is run. The default value is server/solr. @echo. @@ -236,7 +243,7 @@ goto done @echo. @echo -c collection Collection to run healthcheck against. @echo. -@echo -z zkHost ZooKeeper connection string; default is localhost:9983 +@echo -z zkHost Zookeeper connection string; default is localhost:9983 @echo. goto done @@ -262,12 +269,12 @@ echo Usage: solr delete [-c name] [-deleteConfig boolean] [-p port] echo. echo Deletes a core or collection depending on whether Solr is running in standalone (core) or SolrCloud echo mode (collection). If you're deleting a collection in SolrCloud mode, the default behavior is to also -echo delete the configuration directory from ZooKeeper so long as it is not being used by another collection. +echo delete the configuration directory from Zookeeper so long as it is not being used by another collection. echo You can override this behavior by passing -deleteConfig false when running this command. echo. echo -c name Name of core to create echo. -echo -deleteConfig boolean Delete the configuration directory from ZooKeeper; default is true +echo -deleteConfig boolean Delete the configuration directory from Zookeeper; default is true echo. echo -p port Port of a local Solr instance where you want to create the new core echo If not specified, the script will search the local system for a running @@ -317,14 +324,14 @@ echo. echo Alternatively, you can pass the path to your own configuration directory instead of using echo one of the built-in configurations, such as: bin\solr create_collection -c mycoll -d c:/tmp/myconfig echo. -echo By default the script will upload the specified confdir directory into ZooKeeper using the same +echo By default the script will upload the specified confdir directory into Zookeeper using the same echo name as the collection (-c) option. Alternatively, if you want to reuse an existing directory -echo or create a confdir in ZooKeeper that can be shared by multiple collections, use the -n option +echo or create a confdir in Zookeeper that can be shared by multiple collections, use the -n option echo. -echo -n configName Name the configuration directory in ZooKeeper; by default, the configuration -echo will be uploaded to ZooKeeper using the collection name (-c), but if you want +echo -n configName Name the configuration directory in Zookeeper; by default, the configuration +echo will be uploaded to Zookeeper using the collection name (-c), but if you want echo to use an existing directory or override the name of the configuration in -echo ZooKeeper, then use the -c option. +echo Zookeeper, then use the -c option. echo. echo -shards # Number of shards to split the collection into echo. @@ -336,6 +343,30 @@ echo Solr instance and will use the port of the first echo. goto done +:zk_usage +echo. +echo Usage: solr zk [-downconfig or -upconfig] [-d confdir] [-n configName] [-z zkHost] +echo. +echo -upconfig to move a configset from the local machine to Zookeeper +echo. +echo -downconfig to move a configset from Zookeeper to the local machine +echo. +echo -n configName Name of the configset in Zookeeper that will be the destination of +echo 'upconfig' and the source for 'downconfig'. +echo. +echo -d confdir The local directory the configuration will be uploaded from for +echo 'upconfig' or downloaded to for 'downconfig'. For 'upconfig', this +echo can be one of the example configsets, basic_configs, data_driven_schema_configs or +echo sample_techproducts_configs or an arbitrary directory. +echo. +echo -z zkHost Zookeeper connection string. +echo. +echo NOTE: Solr must have been started least once (or have it running) before using this command. +echo This initialized Zookeeper for Solr +echo. +goto done + + REM Really basic command-line arg parsing :parse_args @@ -540,13 +571,13 @@ goto parse_args set "arg=%~2" IF "%arg%"=="" ( - set SCRIPT_ERROR=ZooKeeper connection string is required! + set SCRIPT_ERROR=Zookeeper connection string is required! goto invalid_cmd_line ) set firstChar=%arg:~0,1% IF "%firstChar%"=="-" ( - set SCRIPT_ERROR=Expected ZooKeeper connection string but found %2 instead! + set SCRIPT_ERROR=Expected Zookeeper connection string but found %2 instead! goto invalid_cmd_line ) @@ -783,7 +814,7 @@ IF "%SOLR_MODE%"=="solrcloud" ( IF NOT "%ZK_HOST%"=="" ( set "CLOUD_MODE_OPTS=!CLOUD_MODE_OPTS! -DzkHost=%ZK_HOST%" ) ELSE ( - IF "%verbose%"=="1" echo Configuring SolrCloud to launch an embedded ZooKeeper using -DzkRun + IF "%verbose%"=="1" echo Configuring SolrCloud to launch an embedded Zookeeper using -DzkRun set "CLOUD_MODE_OPTS=!CLOUD_MODE_OPTS! -DzkRun" ) IF EXIST "%SOLR_HOME%\collection1\core.properties" set "CLOUD_MODE_OPTS=!CLOUD_MODE_OPTS! -Dbootstrap_confdir=./solr/collection1/conf -Dcollection.configName=myconf -DnumShards=1" @@ -1143,6 +1174,81 @@ org.apache.solr.util.SolrCLI delete -name !DELETE_NAME! -deleteConfig !DELETE_CO goto done +:parse_zk_args +IF [%1]==[] goto run_zk +IF "%1"=="-upconfig" goto set_zk_op_up +IF "%1"=="-downconfig" goto set_zk_op_down +IF "%1"=="-n" goto set_config_name +IF "%1"=="-configname" goto set_config_name +IF "%1"=="-d" goto set_configdir +IF "%1"=="-confdir" goto set_configdir +IF "%1"=="-z" goto set_config_zk +IF "%1"=="/?" goto usage +IF "%1"=="-h" goto zk_usage +IF "%1"=="-help" goto zk_usage +goto run_zk + +:set_zk_op_up +set ZK_OP=upconfig +SHIFT +goto parse_zk_args + +:set_zk_op_down +set ZK_OP=downconfig +SHIFT +goto parse_zk_args + +:set_config_name +set CONFIGSET_NAME=%~2 +SHIFT +SHIFT +goto parse_zk_args + +:set_configdir +set CONFIGSET_DIR=%~2 +SHIFT +SHIFT +goto parse_zk_args + +:set_config_zk +set CONFIGSET_ZK=%~2 +SHIFT +SHIFT +goto parse_zk_args + + +:run_zk +IF "!ZK_OP!"=="" ( + set "SCRIPT_ERROR=One of '-upconfig' or '-downconfig' is required for %SCRIPT_CMD%" + goto invalid_cmd_line +) + +IF "!CONFIGSET_NAME!"=="" ( + set "SCRIPT_ERROR=Name (-n) is a required parameter for %SCRIPT_CMD%" + goto invalid_cmd_line +) + +if "!CONFIGSET_DIR!"=="" ( + set "SCRIPT_ERROR=Name (-d) is a required parameter for %SCRIPT_CMD%" + goto err +) + +if "!CONFIGSET_ZK!"=="" ( + set "SCRIPT_ERROR=Name (-z) is a required parameter for %SCRIPT_CMD%" + goto err +) + +IF "!ZK_OP!"=="upconfig" ( + "%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^ + -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ + org.apache.solr.util.SolrCLI !ZK_OP! -confname !CONFIGSET_NAME! -confdir !CONFIGSET_DIR! -zkHost !CONFIGSET_ZK! -configsetsDir "%SOLR_TIP%/server/solr/configsets" +) ELSE ( + "%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^ + -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ + org.apache.solr.util.SolrCLI !ZK_OP! -confname !CONFIGSET_NAME! -confdir !CONFIGSET_DIR! -zkHost !CONFIGSET_ZK! +) + +goto done :invalid_cmd_line @echo. @@ -1168,6 +1274,8 @@ IF "%FIRST_ARG%"=="start" ( goto create_core_usage ) ELSE IF "%FIRST_ARG%"=="create_collection" ( goto create_collection_usage +) ELSE IF "%FIRST_ARG%"=="zk" ( + goto zk_usage ) ELSE ( goto script_usage ) diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java index b5f1f558d5d..79fd05aa654 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java @@ -28,11 +28,15 @@ import java.net.Socket; import java.net.SocketException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -150,6 +154,45 @@ public class SolrCLI { } protected abstract void runImpl(CommandLine cli) throws Exception; + + // It's a little awkward putting this in ToolBase, but to re-use it in upconfig and create, _and_ have access + // to the (possibly) redirected "stdout", it needs to go here unless we reorganize things a bit. + protected void upconfig(CloudSolrClient cloudSolrClient, CommandLine cli, String confname, String configSet) throws IOException { + + File configSetDir = null; + // we try to be flexible and allow the user to specify a configuration directory instead of a configset name + File possibleConfigDir = new File(configSet); + if (possibleConfigDir.isDirectory()) { + configSetDir = possibleConfigDir; + } else { + File configsetsDir = new File(cli.getOptionValue("configsetsDir")); + if (!configsetsDir.isDirectory()) + throw new FileNotFoundException(configsetsDir.getAbsolutePath() + " not found!"); + + // upload the configset if it exists + configSetDir = new File(configsetsDir, configSet); + if (!configSetDir.isDirectory()) { + throw new FileNotFoundException("Specified config " + configSet + + " not found in " + configsetsDir.getAbsolutePath()); + } + } + + File confDir = new File(configSetDir, "conf"); + if (!confDir.isDirectory()) { + // config dir should contain a conf sub-directory but if not and there's a solrconfig.xml, then use it + if ((new File(configSetDir, "solrconfig.xml")).isFile()) { + confDir = configSetDir; + } else { + throw new IllegalArgumentException("Specified configuration directory " + configSetDir.getAbsolutePath() + + " is invalid;\nit should contain either conf sub-directory or solrconfig.xml"); + } + } + + // + echo("Uploading " + confDir.getAbsolutePath() + + " for config " + confname + " to ZooKeeper at " + cloudSolrClient.getZkHost()); + cloudSolrClient.uploadConfig(confDir.toPath(), confname); + } } /** @@ -223,7 +266,7 @@ public class SolrCLI { if (args == null || args.length == 0 || args[0] == null || args[0].trim().length() == 0) { System.err.println("Invalid command-line args! Must pass the name of a tool to run.\n" + "Supported tools:\n"); - displayToolOptions(System.err); + displayToolOptions(); exit(1); } @@ -327,6 +370,10 @@ public class SolrCLI { return new ConfigTool(); else if ("run_example".equals(toolType)) return new RunExampleTool(); + else if ("upconfig".equals(toolType)) + return new ConfigSetUploadTool(); + else if ("downconfig".equals(toolType)) + return new ConfigSetDownloadTool(); // If you add a built-in tool to this class, add it here to avoid // classpath scanning @@ -340,7 +387,7 @@ public class SolrCLI { throw new IllegalArgumentException(toolType + " not supported!"); } - private static void displayToolOptions(PrintStream out) throws Exception { + private static void displayToolOptions() throws Exception { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("healthcheck", getToolOptions(new HealthcheckTool())); formatter.printHelp("status", getToolOptions(new StatusTool())); @@ -351,6 +398,8 @@ public class SolrCLI { formatter.printHelp("delete", getToolOptions(new DeleteTool())); formatter.printHelp("config", getToolOptions(new ConfigTool())); formatter.printHelp("run_example", getToolOptions(new RunExampleTool())); + formatter.printHelp("upconfig", getToolOptions(new ConfigSetUploadTool())); + formatter.printHelp("downconfig", getToolOptions(new ConfigSetDownloadTool())); List> toolClasses = findToolClassesInPackage("org.apache.solr.util"); for (Class next : toolClasses) { @@ -1393,40 +1442,7 @@ public class SolrCLI { } else if (configExistsInZk) { echo("Re-using existing configuration directory "+confname); } else { - String configSet = cli.getOptionValue("confdir", DEFAULT_CONFIG_SET); - File configSetDir = null; - // we try to be flexible and allow the user to specify a configuration directory instead of a configset name - File possibleConfigDir = new File(configSet); - if (possibleConfigDir.isDirectory()) { - configSetDir = possibleConfigDir; - } else { - File configsetsDir = new File(cli.getOptionValue("configsetsDir")); - if (!configsetsDir.isDirectory()) - throw new FileNotFoundException(configsetsDir.getAbsolutePath()+" not found!"); - - // upload the configset if it exists - configSetDir = new File(configsetsDir, configSet); - if (!configSetDir.isDirectory()) { - throw new FileNotFoundException("Specified config " + configSet + - " not found in " + configsetsDir.getAbsolutePath()); - } - } - - File confDir = new File(configSetDir, "conf"); - if (!confDir.isDirectory()) { - // config dir should contain a conf sub-directory but if not and there's a solrconfig.xml, then use it - if ((new File(configSetDir, "solrconfig.xml")).isFile()) { - confDir = configSetDir; - } else { - throw new IllegalArgumentException("Specified configuration directory "+configSetDir.getAbsolutePath()+ - " is invalid;\nit should contain either conf sub-directory or solrconfig.xml"); - } - } - - // test to see if that config exists in ZK - echo("Uploading "+confDir.getAbsolutePath()+ - " for config "+confname+" to ZooKeeper at "+cloudSolrClient.getZkHost()); - cloudSolrClient.uploadConfig(confDir.toPath(), confname); + upconfig(cloudSolrClient, cli, confname, cli.getOptionValue("confdir", DEFAULT_CONFIG_SET)); } // since creating a collection is a heavy-weight operation, check for existence first @@ -1639,6 +1655,139 @@ public class SolrCLI { } // end CreateTool class + public static class ConfigSetUploadTool extends ToolBase { + + public ConfigSetUploadTool() { + this(System.out); + } + + public ConfigSetUploadTool(PrintStream stdout) { + super(stdout); + } + + @SuppressWarnings("static-access") + public Option[] getOptions() { + return new Option[]{ + OptionBuilder + .withArgName("confname") // Comes out in help message + .hasArg() // Has one sub-argument + .isRequired(true) // confname argument must be present + .withDescription("Configset name on Zookeeper") + .create("confname"), // passed as -confname value + OptionBuilder + .withArgName("confdir") + .hasArg() + .isRequired(true) + .withDescription("Local directory with configs") + .create("confdir"), + OptionBuilder + .withArgName("configsetsDir") + .hasArg() + .isRequired() + .withDescription("Parent directory of example configsets") + .create("configsetsDir"), + OptionBuilder + .withArgName("HOST") + .hasArg() + .isRequired(true) + .withDescription("Address of the Zookeeper ensemble; defaults to: " + ZK_HOST) + .create("zkHost") + }; + } + + + public String getName() { + return "upconfig"; + } + + protected void runImpl(CommandLine cli) throws Exception { + String zkHost = getZkHost(cli); + if (zkHost == null) { + throw new IllegalStateException("Solr at " + cli.getOptionValue("solrUrl") + + " is running in standalone server mode, upconfig can only be used when running in SolrCloud mode.\n"); + } + + try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + echo("\nConnecting to ZooKeeper at " + zkHost + " ..."); + cloudSolrClient.connect(); + upconfig(cloudSolrClient, cli, cli.getOptionValue("confname"), cli.getOptionValue("confdir")); + } + } + } + + public static class ConfigSetDownloadTool extends ToolBase { + + public ConfigSetDownloadTool() { + this(System.out); + } + + public ConfigSetDownloadTool(PrintStream stdout) { + super(stdout); + } + + @SuppressWarnings("static-access") + public Option[] getOptions() { + return new Option[]{ + OptionBuilder + .withArgName("confname") + .hasArg() + .isRequired(true) + .withDescription("Configset name on Zookeeper") + .create("confname"), + OptionBuilder + .withArgName("confdir") + .hasArg() + .isRequired(true) + .withDescription("Local directory with configs") + .create("confdir"), + OptionBuilder + .withArgName("HOST") + .hasArg() + .isRequired(true) + .withDescription("Address of the Zookeeper ensemble; defaults to: " + ZK_HOST) + .create("zkHost") + }; + } + + public String getName() { + return "downconfig"; + } + + protected void runImpl(CommandLine cli) throws Exception { + + String zkHost = getZkHost(cli); + if (zkHost == null) { + throw new IllegalStateException("Solr at " + cli.getOptionValue("solrUrl") + + " is running in standalone server mode, downconfig can only be used when running in SolrCloud mode.\n"); + } + + + try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { + echo("\nConnecting to ZooKeeper at " + zkHost + " ..."); + cloudSolrClient.connect(); + downconfig(cloudSolrClient, cli.getOptionValue("confname"), cli.getOptionValue("confdir")); + } + } + + protected void downconfig(CloudSolrClient cloudSolrClient, String confname, String confdir) throws IOException { + + Path configSetPath = Paths.get(confdir); + // we try to be nice about having the "conf" in the directory, and we create it if it's not there. + if (configSetPath.endsWith("/conf") == false) { + configSetPath = Paths.get(configSetPath.toString(), "conf"); + } + if (Files.exists(configSetPath) == false) { + Files.createDirectories(configSetPath); + } + + // Try to download the configset + echo("Downloading configset " + confname + " from ZooKeeper at " + cloudSolrClient.getZkHost() + + " to directory " + configSetPath.toAbsolutePath()); + cloudSolrClient.downloadConfig(confname, configSetPath); + } + + } // End ConfigSetDownloadTool class + public static class DeleteTool extends ToolBase { public DeleteTool() { this(System.out); } @@ -2486,13 +2635,13 @@ public class SolrCLI { echo("Please choose a configuration for the "+collectionName+" collection, available options are:"); cloudConfig = - prompt(readInput, "basic_configs, data_driven_schema_configs, or sample_techproducts_configs ["+cloudConfig+"] ", cloudConfig); + prompt(readInput, "basic_configs, data_driven_schema_configs, sample_techproducts_configs, or managed_schema_configs ["+cloudConfig+"] ", cloudConfig); // validate the cloudConfig name while (!isValidConfig(configsetsDir, cloudConfig)) { echo(cloudConfig+" is not a valid configuration directory! Please choose a configuration for the "+collectionName+" collection, available options are:"); cloudConfig = - prompt(readInput, "basic_configs, data_driven_schema_configs, or sample_techproducts_configs ["+cloudConfig+"] ", cloudConfig); + prompt(readInput, "basic_configs, data_driven_schema_configs, sample_techproducts_configs, or managed_schema_configs ["+cloudConfig+"] ", cloudConfig); } } else { // must verify if default collection exists