From c1522a9c752e679291476a9b7a9c323a4c83c97a Mon Sep 17 00:00:00 2001 From: Timothy Potter Date: Tue, 4 Aug 2015 16:32:12 +0000 Subject: [PATCH] SOLR-7847: Implement run example logic in Java instead of OS-specific scripts in bin/solr git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1694083 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/ivy-versions.properties | 1 + solr/CHANGES.txt | 3 + solr/bin/solr | 316 +--- solr/bin/solr.cmd | 251 +-- solr/core/ivy.xml | 1 + .../java/org/apache/solr/util/SolrCLI.java | 1365 +++++++++++++---- .../solr/cloud/SolrCloudExampleTest.java | 7 +- .../solr/util/TestSolrCLIRunExample.java | 485 ++++++ solr/licenses/commons-exec-1.3.jar.sha1 | 1 + solr/licenses/commons-exec-LICENSE-ASL.txt | 203 +++ solr/licenses/commons-exec-NOTICE.txt | 5 + 11 files changed, 1846 insertions(+), 792 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java create mode 100644 solr/licenses/commons-exec-1.3.jar.sha1 create mode 100644 solr/licenses/commons-exec-LICENSE-ASL.txt create mode 100644 solr/licenses/commons-exec-NOTICE.txt diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 08639abed16..9f8fa345d21 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -96,6 +96,7 @@ com.sun.jersey.version = 1.9 /org.apache.ant/ant = 1.8.2 /org.apache.avro/avro = 1.7.5 /org.apache.commons/commons-compress = 1.8.1 +/org.apache.commons/commons-exec = 1.3 /org.apache.commons/commons-math3 = 3.4.1 org.apache.curator.version = 2.8.0 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2fd44cf4966..1ef55d2a499 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -405,6 +405,9 @@ Other Changes * SOLR-7832: bin/post now allows either -url or -c, rather than requiring both. (ehatcher) +* SOLR-7847: Implement run example logic in Java instead of OS-specific scripts in + bin/solr and bin\solr.cmd (Timothy Potter) + ================== 5.2.1 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/bin/solr b/solr/bin/solr index f46251f7079..c0a84e65f17 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -809,12 +809,14 @@ fi FG="false" noprompt=false SOLR_OPTS=($SOLR_OPTS) +PASS_TO_RUN_EXAMPLE= if [ $# -gt 0 ]; then while true; do case "$1" in -c|-cloud) SOLR_MODE="solrcloud" + PASS_TO_RUN_EXAMPLE+=" -c" shift ;; -d|-dir) @@ -864,6 +866,7 @@ if [ $# -gt 0 ]; then exit 1 fi SOLR_HOST="$2" + PASS_TO_RUN_EXAMPLE+=" -h $SOLR_HOST" shift 2 ;; -m|-memory) @@ -872,6 +875,7 @@ if [ $# -gt 0 ]; then exit 1 fi SOLR_HEAP="$2" + PASS_TO_RUN_EXAMPLE+=" -m $SOLR_HEAP" shift 2 ;; -p|-port) @@ -880,6 +884,7 @@ if [ $# -gt 0 ]; then exit 1 fi SOLR_PORT="$2" + PASS_TO_RUN_EXAMPLE+=" -p $SOLR_PORT" shift 2 ;; -z|-zkhost) @@ -889,10 +894,12 @@ if [ $# -gt 0 ]; then fi ZK_HOST="$2" SOLR_MODE="solrcloud" + PASS_TO_RUN_EXAMPLE+=" -z $ZK_HOST" shift 2 ;; -a|-addlopts) ADDITIONAL_CMD_OPTS="$2" + PASS_TO_RUN_EXAMPLE+=" -a \"$ADDITIONAL_CMD_OPTS\"" shift 2 ;; -k|-key) @@ -905,10 +912,12 @@ if [ $# -gt 0 ]; then ;; -noprompt) noprompt=true + PASS_TO_RUN_EXAMPLE+=" -noprompt" shift ;; -V|-verbose) verbose=true + PASS_TO_RUN_EXAMPLE+=" --verbose" shift ;; -all) @@ -923,6 +932,7 @@ if [ $# -gt 0 ]; then if [ "${1:0:2}" == "-D" ]; then # pass thru any opts that begin with -D (java system props) SOLR_OPTS+=("$1") + PASS_TO_RUN_EXAMPLE+=" $1" shift else if [ "$1" != "" ]; then @@ -937,6 +947,31 @@ if [ $# -gt 0 ]; then done fi +if [ -z "$SOLR_SERVER_DIR" ]; then + SOLR_SERVER_DIR="$DEFAULT_SERVER_DIR" +fi + +if [ ! -e "$SOLR_SERVER_DIR" ]; then + echo -e "\nSolr server directory $SOLR_SERVER_DIR not found!\n" + exit 1 +fi + +if [[ "$FG" == 'true' && "$EXAMPLE" != "" ]]; then + FG='false' + echo -e "\nWARNING: Foreground mode (-f) not supported when running examples.\n" +fi + +# +# If the user specified an example to run, invoke the run_example tool (Java app) and exit +# otherwise let this script proceed to process the user request +# +if [ -n "$EXAMPLE" ] && [ "$SCRIPT_CMD" == "start" ]; then + run_tool run_example -e $EXAMPLE -d "$SOLR_SERVER_DIR" -urlScheme $SOLR_URL_SCHEME $PASS_TO_RUN_EXAMPLE + exit $? +fi + +############# start/stop logic below here ################ + if $verbose ; then echo "Using Solr root directory: $SOLR_TIP" echo "Using Java: $JAVA" @@ -949,137 +984,6 @@ else SOLR_HOST_ARG=() fi -if [ -z "$SOLR_SERVER_DIR" ]; then - SOLR_SERVER_DIR="$DEFAULT_SERVER_DIR" -fi - -if [ ! -e "$SOLR_SERVER_DIR" ]; then - echo -e "\nSolr server directory $SOLR_SERVER_DIR not found!\n" - exit 1 -fi - -CLOUD_NUM_NODES=2 -declare -a CLOUD_PORTS=('8983' '7574' '8984' '7575'); - -# select solr.solr.home based on the desired example -if [ "$EXAMPLE" != "" ]; then - case $EXAMPLE in - cloud) - # - # Engage in an interactive session with user to setup the SolrCloud example - # - echo -e "\nWelcome to the SolrCloud example!\n\n" - if $noprompt ; then - CLOUD_NUM_NODES=2 - echo -e "Starting up $CLOUD_NUM_NODES Solr nodes for your example SolrCloud cluster." - else - echo -e "This interactive session will help you launch a SolrCloud cluster on your local workstation.\n" - read -e -p "To begin, how many Solr nodes would you like to run in your local cluster? (specify 1-4 nodes) [2] " USER_INPUT - while true - do - CLOUD_NUM_NODES=`echo $USER_INPUT | tr -d ' '` - if [ -z "$CLOUD_NUM_NODES" ]; then - CLOUD_NUM_NODES=2 - fi - if [[ $CLOUD_NUM_NODES > 4 || $CLOUD_NUM_NODES < 1 ]]; then - read -e -p "Please provide a node count between 1 and 4 [2] " USER_INPUT - else - break; - fi - done - - echo -e "Ok, let's start up $CLOUD_NUM_NODES Solr nodes for your example SolrCloud cluster.\n" - for (( s=0; s<$CLOUD_NUM_NODES; s++ )) - do - read -e -p "Please enter the port for node$[$s+1] [${CLOUD_PORTS[$s]}] " USER_INPUT - while true - do - # trim whitespace out of the user input - CLOUD_PORT=`echo $USER_INPUT | tr -d ' '` - - # handle the default selection or empty input - if [ -z "$CLOUD_PORT" ]; then - CLOUD_PORT=${CLOUD_PORTS[$s]} - fi - - # check to see if something is already bound to that port - if hash lsof 2>/dev/null ; then # hash returns true if lsof is on the path - PORT_IN_USE=`lsof -PniTCP:$CLOUD_PORT -sTCP:LISTEN` - if [ "$PORT_IN_USE" != "" ]; then - read -e -p "Oops! Looks like port $CLOUD_PORT is already being used by another process. Please choose a different port. " USER_INPUT - else - CLOUD_PORTS[$s]=$CLOUD_PORT - echo $CLOUD_PORT - break; - fi - else - CLOUD_PORTS[$s]=$CLOUD_PORT - echo $CLOUD_PORT - break; - fi - done - done - fi - - # setup a unqiue solr.solr.home directory for each node - CLOUD_EXAMPLE_DIR="$SOLR_TIP/example/cloud" - if [ ! -d "$CLOUD_EXAMPLE_DIR/node1/solr" ]; then - echo "Creating Solr home directory $CLOUD_EXAMPLE_DIR/node1/solr" - mkdir -p "$CLOUD_EXAMPLE_DIR/node1/solr" - cp "$DEFAULT_SERVER_DIR/solr/solr.xml" "$CLOUD_EXAMPLE_DIR/node1/solr/" - cp "$DEFAULT_SERVER_DIR/solr/zoo.cfg" "$CLOUD_EXAMPLE_DIR/node1/solr/" - fi - - for (( s=1; s<$CLOUD_NUM_NODES; s++ )) - do - ndx=$[$s+1] - if [ ! -d "$CLOUD_EXAMPLE_DIR/node$ndx" ]; then - echo "Cloning Solr home directory $CLOUD_EXAMPLE_DIR/node1 into $CLOUD_EXAMPLE_DIR/node$ndx" - cp -r "$CLOUD_EXAMPLE_DIR/node1" "$CLOUD_EXAMPLE_DIR/node$ndx" - fi - done - SOLR_MODE="solrcloud" - SOLR_SERVER_DIR="$SOLR_TIP/server" - SOLR_HOME="$CLOUD_EXAMPLE_DIR/node1/solr" - SOLR_PORT="${CLOUD_PORTS[0]}" - shift - ;; - techproducts) - SOLR_HOME="$SOLR_TIP/example/techproducts/solr" - mkdir -p "$SOLR_HOME" - if [ ! -f "$SOLR_HOME/solr.xml" ]; then - cp "$DEFAULT_SERVER_DIR/solr/solr.xml" "$SOLR_HOME/solr.xml" - cp "$DEFAULT_SERVER_DIR/solr/zoo.cfg" "$SOLR_HOME/zoo.cfg" - fi - EXAMPLE_CONFIGSET='sample_techproducts_configs' - shift - ;; - dih) - SOLR_HOME="$SOLR_TIP/example/example-DIH/solr" - shift - ;; - schemaless) - SOLR_HOME="$SOLR_TIP/example/schemaless/solr" - mkdir -p "$SOLR_HOME" - if [ ! -f "$SOLR_HOME/solr.xml" ]; then - cp "$DEFAULT_SERVER_DIR/solr/solr.xml" "$SOLR_HOME/solr.xml" - cp "$DEFAULT_SERVER_DIR/solr/zoo.cfg" "$SOLR_HOME/zoo.cfg" - fi - EXAMPLE_CONFIGSET='data_driven_schema_configs' - shift - ;; - *) - print_usage "start" "Unsupported example $EXAMPLE! Please choose one of: cloud, dih, schemaless, or techproducts" - exit 1 - ;; - esac -fi - -if [[ "$FG" == 'true' && "$EXAMPLE" != "" ]]; then - FG='false' - echo -e "\nWARNING: Foreground mode (-f) not supported when running examples.\n" -fi - if [ -z "$STOP_KEY" ]; then STOP_KEY='solrrocks' fi @@ -1409,7 +1313,7 @@ function launch_solr() { # no lsof on cygwin though if hash lsof 2>/dev/null ; then # hash returns true if lsof is on the path - echo -n "Waiting to see Solr listening on port $SOLR_PORT" + echo -n "Waiting up to 30 seconds to see Solr running on port $SOLR_PORT" # Launch in a subshell to show the spinner (loops=0 while true @@ -1441,152 +1345,6 @@ function launch_solr() { fi } -if [ "$EXAMPLE" != "cloud" ]; then - launch_solr "$FG" "$ADDITIONAL_CMD_OPTS" - - # create the core/collection for the requested example after launching Solr - if [[ "$EXAMPLE" == "schemaless" || "$EXAMPLE" == "techproducts" ]]; then - - if [ "$EXAMPLE" == "schemaless" ]; then - EXAMPLE_NAME=gettingstarted - else - EXAMPLE_NAME="$EXAMPLE" - fi - - run_tool create -name "$EXAMPLE_NAME" -shards 1 -replicationFactor 1 \ - -confname "$EXAMPLE_NAME" -confdir "$EXAMPLE_CONFIGSET" \ - -configsetsDir "$SOLR_TIP/server/solr/configsets" -solrUrl $SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr - if [ $? -ne 0 ]; then - exit 1 - fi - - if [ "$EXAMPLE" == "techproducts" ]; then - echo "Indexing tech product example docs from $SOLR_TIP/example/exampledocs" - "$JAVA" $SOLR_SSL_OPTS -Durl="$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr/$EXAMPLE/update" \ - -jar "$SOLR_TIP/example/exampledocs/post.jar" "$SOLR_TIP/example/exampledocs"/*.xml - fi - - echo -e "\nSolr $EXAMPLE example launched successfully. Direct your Web browser to $SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr to visit the Solr Admin UI\n" - fi -else - # - # SolrCloud example is a bit involved so needs special handling here - # - SOLR_SERVER_DIR="$SOLR_TIP/server" - SOLR_HOME="$SOLR_TIP/example/cloud/node1/solr" - SOLR_PORT="${CLOUD_PORTS[0]}" - - if [ "$ZK_HOST" != "" ]; then - DASHZ="-z $ZK_HOST" - fi - - if [ "$SOLR_HEAP" != "" ]; then - DASHM="-m $SOLR_HEAP" - fi - - if [ "$ADDITIONAL_CMD_OPTS" != "" ]; then - DASHA="-a $ADDITIONAL_CMD_OPTS" - fi - - echo -e "\nStarting up SolrCloud node1 on port ${CLOUD_PORTS[0]} using command:\n" - echo -e "solr start -cloud -s example/cloud/node1/solr -p $SOLR_PORT $DASHZ $DASHM $DASHA\n\n" - - # can't launch this node in the foreground else we can't run anymore commands - launch_solr "false" "$ADDITIONAL_CMD_OPTS" - - # if user did not define a specific -z parameter, assume embedded in first cloud node we launched above - zk_host="$ZK_HOST" - if [ -z "$zk_host" ]; then - zk_port=$[$SOLR_PORT+1000] - zk_host="localhost:$zk_port" - fi - - for (( s=1; s<$CLOUD_NUM_NODES; s++ )) - do - ndx=$[$s+1] - next_port="${CLOUD_PORTS[$s]}" - echo -e "\n\nStarting node$ndx on port $next_port using command:\n" - echo -e "solr start -cloud -s example/cloud/node$ndx/solr -p $next_port -z $zk_host $DASHM $DASHA \n\n" - # call this script again with correct args for next node - "$SOLR_TIP/bin/solr" start -cloud -s "$SOLR_TIP/example/cloud/node$ndx/solr" -p "$next_port" -z "$zk_host" $DASHM $DASHA - done - - # TODO: better (shorter) name?? - CLOUD_COLLECTION='gettingstarted' - - if $noprompt ; then - CLOUD_NUM_SHARDS=2 - CLOUD_REPFACT=2 - CLOUD_CONFIG='data_driven_schema_configs' - else - echo -e "\nNow let's create a new collection for indexing documents in your $CLOUD_NUM_NODES-node cluster.\n" - read -e -p "Please provide a name for your new collection: [gettingstarted] " USER_INPUT - # trim whitespace out of the user input - CLOUD_COLLECTION=`echo "$USER_INPUT" | tr -d ' '` - - # handle the default selection or empty input - if [ -z "$CLOUD_COLLECTION" ]; then - CLOUD_COLLECTION='gettingstarted' - fi - echo $CLOUD_COLLECTION - - USER_INPUT= - read -e -p "How many shards would you like to split $CLOUD_COLLECTION into? [2] " USER_INPUT - # trim whitespace out of the user input - CLOUD_NUM_SHARDS=`echo "$USER_INPUT" | tr -d ' '` - - # handle the default selection or empty input - if [ -z "$CLOUD_NUM_SHARDS" ]; then - CLOUD_NUM_SHARDS=2 - fi - echo $CLOUD_NUM_SHARDS - - USER_INPUT= - read -e -p "How many replicas per shard would you like to create? [2] " USER_INPUT - # trim whitespace out of the user input - CLOUD_REPFACT=`echo $USER_INPUT | tr -d ' '` - - # handle the default selection or empty input - if [ -z "$CLOUD_REPFACT" ]; then - CLOUD_REPFACT=2 - fi - echo $CLOUD_REPFACT - - USER_INPUT= - echo "Please choose a configuration for the $CLOUD_COLLECTION collection, available options are:" - read -e -p "basic_configs, data_driven_schema_configs, or sample_techproducts_configs [data_driven_schema_configs] " USER_INPUT - while true - do - # trim whitespace out of the user input - CLOUD_CONFIG=`echo "$USER_INPUT" | tr -d ' '` - - # handle the default selection or empty input - if [ -z "$CLOUD_CONFIG" ]; then - CLOUD_CONFIG='data_driven_schema_configs' - fi - - # validate the confdir arg - if [[ ! -d "$SOLR_TIP/server/solr/configsets/$CLOUD_CONFIG" && ! -d "$CLOUD_CONFIG" ]]; then - echo -e "\nOops! Specified configuration $CLOUD_CONFIG not found!" - read -e -p "Choose one of: basic_configs, data_driven_schema_configs, or sample_techproducts_configs [data_driven_schema_configs] " USER_INPUT - CLOUD_CONFIG= - else - break; - fi - done - - fi - - run_tool create_collection -name "$CLOUD_COLLECTION" -shards $CLOUD_NUM_SHARDS -replicationFactor $CLOUD_REPFACT \ - -confname "$CLOUD_COLLECTION" -confdir "$CLOUD_CONFIG" \ - -configsetsDir "$SOLR_TIP/server/solr/configsets" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr" - - # enable soft-autocommits for the gettingstarted collection - echo -e "\nEnabling auto soft-commits with maxTime 3 secs using the Config API" - run_tool config -collection "$CLOUD_COLLECTION" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr" \ - -property updateHandler.autoSoftCommit.maxTime -value 3000 - - echo -e "\n\nSolrCloud example running, please visit $SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$SOLR_PORT/solr \n\n" -fi +launch_solr "$FG" "$ADDITIONAL_CMD_OPTS" exit $? diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 04d27563ce1..b6cd308109c 100644 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -18,6 +18,8 @@ IF "%OS%"=="Windows_NT" setlocal enabledelayedexpansion enableextensions +set "PASS_TO_RUN_EXAMPLE=" + REM Determine top-level Solr directory set SDIR=%~dp0 IF "%SDIR:~-1%"=="\" set SDIR=%SDIR:~0,-1% @@ -378,6 +380,7 @@ goto parse_args :set_verbose set verbose=1 +set "PASS_TO_RUN_EXAMPLE=--verbose !PASS_TO_RUN_EXAMPLE!" SHIFT goto parse_args @@ -461,6 +464,7 @@ IF "%firstChar%"=="-" ( ) set SOLR_HEAP=%~2 +set "PASS_TO_RUN_EXAMPLE=-m %~2 !PASS_TO_RUN_EXAMPLE!" SHIFT SHIFT goto parse_args @@ -479,6 +483,7 @@ IF "%firstChar%"=="-" ( ) set SOLR_HOST=%~2 +set "PASS_TO_RUN_EXAMPLE=-h %~2 !PASS_TO_RUN_EXAMPLE!" SHIFT SHIFT goto parse_args @@ -497,6 +502,7 @@ IF "%firstChar%"=="-" ( ) set SOLR_PORT=%~2 +set "PASS_TO_RUN_EXAMPLE=-p %~2 !PASS_TO_RUN_EXAMPLE!" SHIFT SHIFT goto parse_args @@ -538,6 +544,7 @@ IF "%firstChar%"=="-" ( ) set "ZK_HOST=%~2" +set "PASS_TO_RUN_EXAMPLE=-z %~2 !PASS_TO_RUN_EXAMPLE!" SHIFT SHIFT goto parse_args @@ -556,12 +563,15 @@ IF NOT "%SOLR_OPTS%"=="" ( ) ELSE ( set "SOLR_OPTS=%PASSTHRU%" ) +set "PASS_TO_RUN_EXAMPLE=%PASSTHRU% !PASS_TO_RUN_EXAMPLE!" SHIFT SHIFT goto parse_args :set_noprompt set NO_USER_PROMPT=1 +set "PASS_TO_RUN_EXAMPLE=-noprompt !PASS_TO_RUN_EXAMPLE!" + SHIFT goto parse_args @@ -588,37 +598,7 @@ IF NOT EXIST "%SOLR_SERVER_DIR%" ( goto err ) -IF "%EXAMPLE%"=="" ( - REM SOLR_HOME just becomes serverDir/solr -) ELSE IF "%EXAMPLE%"=="techproducts" ( - mkdir "%SOLR_TIP%\example\techproducts\solr" - set "SOLR_HOME=%SOLR_TIP%\example\techproducts\solr" - IF NOT EXIST "!SOLR_HOME!\solr.xml" ( - copy "%DEFAULT_SERVER_DIR%\solr\solr.xml" "!SOLR_HOME!\solr.xml" - ) - IF NOT EXIST "!SOLR_HOME!\zoo.cfg" ( - copy "%DEFAULT_SERVER_DIR%\solr\zoo.cfg" "!SOLR_HOME!\zoo.cfg" - ) -) ELSE IF "%EXAMPLE%"=="cloud" ( - set SOLR_MODE=solrcloud - goto cloud_example_start -) ELSE IF "%EXAMPLE%"=="dih" ( - set "SOLR_HOME=%SOLR_TIP%\example\example-DIH\solr" -) ELSE IF "%EXAMPLE%"=="schemaless" ( - mkdir "%SOLR_TIP%\example\schemaless\solr" - set "SOLR_HOME=%SOLR_TIP%\example\schemaless\solr" - IF NOT EXIST "!SOLR_HOME!\solr.xml" ( - copy "%DEFAULT_SERVER_DIR%\solr\solr.xml" "!SOLR_HOME!\solr.xml" - ) - IF NOT EXIST "!SOLR_HOME!\zoo.cfg" ( - copy "%DEFAULT_SERVER_DIR%\solr\zoo.cfg" "!SOLR_HOME!\zoo.cfg" - ) -) ELSE ( - @echo. - @echo 'Unrecognized example %EXAMPLE%!' - @echo. - goto start_usage -) +IF NOT "%EXAMPLE%"=="" goto run_example :start_solr IF "%SOLR_HOME%"=="" set "SOLR_HOME=%SOLR_SERVER_DIR%\solr" @@ -900,9 +880,6 @@ IF "%JAVA_VENDOR%" == "IBM J9" ( set "GCLOG_OPT=-Xloggc" ) -@echo. -CALL :safe_echo "Starting Solr on port %SOLR_PORT% from %SOLR_SERVER_DIR%" -@echo. IF "%FG%"=="1" ( REM run solr in the foreground title "Solr-%SOLR_PORT%" @@ -913,211 +890,25 @@ IF "%FG%"=="1" ( START /B "Solr-%SOLR_PORT%" /D "%SOLR_SERVER_DIR%" "%JAVA%" %SERVEROPT% -Xss256k %SOLR_JAVA_MEM% %START_OPTS% %GCLOG_OPT%:"!SOLR_LOGS_DIR!"/solr_gc.log -Dlog4j.configuration="%LOG4J_CONFIG%" -DSTOP.PORT=!STOP_PORT! -DSTOP.KEY=%STOP_KEY% ^ -Djetty.port=%SOLR_PORT% -Dsolr.solr.home="%SOLR_HOME%" -Dsolr.install.dir="%SOLR_TIP%" -Djetty.home="%SOLR_SERVER_DIR%" -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar "%SOLR_JETTY_CONFIG%" > "!SOLR_LOGS_DIR!\solr-%SOLR_PORT%-console.log" echo %SOLR_PORT%>"%SOLR_TIP%"\bin\solr-%SOLR_PORT%.port -) -set EXAMPLE_NAME=%EXAMPLE% -set CREATE_EXAMPLE_CONFIG= -IF "%EXAMPLE%"=="schemaless" ( - set EXAMPLE_NAME=gettingstarted - set CREATE_EXAMPLE_CONFIG=data_driven_schema_configs + REM now wait to see Solr come online ... + "%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^ + -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ + org.apache.solr.util.SolrCLI status -maxWaitSecs 30 -solr !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%SOLR_PORT%/solr ) -IF "%EXAMPLE%"=="techproducts" ( - set CREATE_EXAMPLE_CONFIG=sample_techproducts_configs -) - -IF NOT "!CREATE_EXAMPLE_CONFIG!"=="" ( - timeout /T 10 - IF "%SOLR_MODE%"=="solrcloud" ( - "%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^ - -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ - org.apache.solr.util.SolrCLI create_collection -name !EXAMPLE_NAME! -shards 1 -replicationFactor 1 ^ - -confdir !CREATE_EXAMPLE_CONFIG! -configsetsDir "%SOLR_SERVER_DIR%\solr\configsets" -solrUrl !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%SOLR_PORT%/solr - ) ELSE ( - "%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^ - -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ - org.apache.solr.util.SolrCLI create_core -name !EXAMPLE_NAME! -solrUrl !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%SOLR_PORT%/solr ^ - -confdir !CREATE_EXAMPLE_CONFIG! -configsetsDir "%SOLR_SERVER_DIR%\solr\configsets" - ) -) - -IF "%EXAMPLE%"=="techproducts" ( - @echo. - @echo Indexing tech product example docs from "%SOLR_TIP%\example\exampledocs" - "%JAVA%" %SOLR_SSL_OPTS% -Durl=!SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%SOLR_PORT%/solr/%EXAMPLE%/update -jar "%SOLR_TIP%/example/exampledocs/post.jar" "%SOLR_TIP%/example/exampledocs/*.xml" -) - -@echo. -IF NOT "%EXAMPLE%"=="" ( - @echo Solr %EXAMPLE% example launched successfully. -) -@echo Direct your Web browser to !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%SOLR_PORT%/solr to visit the Solr Admin UI -@echo. goto done -:cloud_example_start -REM Launch interactive session to guide the user through the SolrCloud example +:run_example +REM Run the requested example -CLS -@echo. -@echo Welcome to the SolrCloud example -@echo. -@echo. - -IF "%NO_USER_PROMPT%"=="1" ( - set CLOUD_NUM_NODES=2 - @echo Starting up %CLOUD_NUM_NODES% Solr nodes for your example SolrCloud cluster. - goto start_cloud_nodes -) ELSE ( - @echo This interactive session will help you launch a SolrCloud cluster on your local workstation. - @echo. - SET /P "USER_INPUT=To begin, how many Solr nodes would you like to run in your local cluster (specify 1-4 nodes) [2]: " - goto while_num_nodes_not_valid -) - -:while_num_nodes_not_valid -IF "%USER_INPUT%"=="" set USER_INPUT=2 -SET /A INPUT_AS_NUM=!USER_INPUT!*1 -IF %INPUT_AS_NUM% GEQ 1 IF %INPUT_AS_NUM% LEQ 4 set CLOUD_NUM_NODES=%INPUT_AS_NUM% -IF NOT DEFINED CLOUD_NUM_NODES ( - SET USER_INPUT= - SET /P "USER_INPUT=Please enter a number between 1 and 4 [2]: " - goto while_num_nodes_not_valid -) -@echo Ok, let's start up %CLOUD_NUM_NODES% Solr nodes for your example SolrCloud cluster. - -:start_cloud_nodes - -set "CLOUD_EXAMPLE_DIR=%SOLR_TIP%\example\cloud" - -@echo Creating Solr home "%CLOUD_EXAMPLE_DIR%\node1\solr" -mkdir "%CLOUD_EXAMPLE_DIR%\node1\solr" -copy "%DEFAULT_SERVER_DIR%\solr\solr.xml" "%CLOUD_EXAMPLE_DIR%\node1\solr\solr.xml" -copy "%DEFAULT_SERVER_DIR%\solr\zoo.cfg" "%CLOUD_EXAMPLE_DIR%\node1\solr\zoo.cfg" - -for /l %%x in (2, 1, !CLOUD_NUM_NODES!) do ( - IF NOT EXIST "%SOLR_TIP%\node%%x" ( - @echo Cloning "%CLOUD_EXAMPLE_DIR%\node1" into "%CLOUD_EXAMPLE_DIR%\node%%x" - xcopy /Q /E /I "%CLOUD_EXAMPLE_DIR%\node1" "%CLOUD_EXAMPLE_DIR%\node%%x" - ) -) - -for /l %%x in (1, 1, !CLOUD_NUM_NODES!) do ( - set USER_INPUT= - set /A idx=%%x-1 - set DEF_PORT=8983 - IF %%x EQU 2 ( - set DEF_PORT=7574 - ) ELSE ( - IF %%x EQU 3 ( - set DEF_PORT=8984 - ) ELSE ( - IF %%x EQU 4 ( - set DEF_PORT=7575 - ) - ) - ) - - IF "%NO_USER_PROMPT%"=="1" ( - set NODE_PORT=!DEF_PORT! - ) ELSE ( - set /P "USER_INPUT=Please enter the port for node%%x [!DEF_PORT!]: " - IF "!USER_INPUT!"=="" set USER_INPUT=!DEF_PORT! - set NODE_PORT=!USER_INPUT! - echo node%%x port: !NODE_PORT! - @echo. - ) - - IF NOT "!SOLR_HEAP!"=="" ( - set "DASHM=-m !SOLR_HEAP!" - ) ELSE ( - set "DASHM=" - ) - - IF %%x EQU 1 ( - set EXAMPLE= - IF NOT "!ZK_HOST!"=="" ( - set "DASHZ=-z !ZK_HOST!" - ) ELSE ( - set "DASHZ=" - ) - @echo Starting node1 on port !NODE_PORT! using command: - @echo solr -cloud -p !NODE_PORT! -s example\node1\solr !DASHZ! !DASHM! - START "Solr-!NODE_PORT!" /D "%SDIR%" solr -f -cloud -p !NODE_PORT! !DASHZ! !DASHM! -s "%CLOUD_EXAMPLE_DIR%\node1\solr" - set NODE1_PORT=!NODE_PORT! - echo !NODE_PORT!>"%SOLR_TIP%"\bin\solr-!NODE_PORT!.port - ) ELSE ( - IF "!ZK_HOST!"=="" ( - set /A ZK_PORT=!NODE1_PORT!+1000 - set "ZK_HOST=localhost:!ZK_PORT!" - ) - @echo Starting node%%x on port !NODE_PORT! using command: - @echo solr -cloud -p !NODE_PORT! -s example\node%%x\solr -z !ZK_HOST! !DASHM! - START "Solr-!NODE_PORT!" /D "%SDIR%" solr -f -cloud -p !NODE_PORT! -z !ZK_HOST! !DASHM! -s "%CLOUD_EXAMPLE_DIR%\node%%x\solr" - echo !NODE_PORT!>"%SOLR_TIP%"\bin\solr-!NODE_PORT!.port - ) - - timeout /T 10 -) - -set USER_INPUT= -echo. -echo Now let's create a new collection for indexing documents in your %CLOUD_NUM_NODES%-node cluster. -IF "%NO_USER_PROMPT%"=="1" ( - set CLOUD_COLLECTION=gettingstarted - set CLOUD_NUM_SHARDS=2 - set CLOUD_REPFACT=2 - set CLOUD_CONFIG=data_driven_schema_configs - goto create_collection -) ELSE ( - goto get_create_collection_params -) - -:get_create_collection_params -set /P "USER_INPUT=Please provide a name for your new collection: [gettingstarted] " -IF "!USER_INPUT!"=="" set USER_INPUT=gettingstarted -set CLOUD_COLLECTION=!USER_INPUT! -echo !CLOUD_COLLECTION! -set USER_INPUT= -echo. -set /P "USER_INPUT=How many shards would you like to split !CLOUD_COLLECTION! into? [2] " -IF "!USER_INPUT!"=="" set USER_INPUT=2 -set CLOUD_NUM_SHARDS=!USER_INPUT! -echo !CLOUD_NUM_SHARDS! -set USER_INPUT= -echo. -set /P "USER_INPUT=How many replicas per shard would you like to create? [2] " -IF "!USER_INPUT!"=="" set USER_INPUT=2 -set CLOUD_REPFACT=!USER_INPUT! -echo !CLOUD_REPFACT! -set USER_INPUT= -echo. -set /P "USER_INPUT=Please choose a configuration for the !CLOUD_COLLECTION! collection, available options are: basic_configs, data_driven_schema_configs, or sample_techproducts_configs [data_driven_schema_configs]" -IF "!USER_INPUT!"=="" set USER_INPUT=data_driven_schema_configs -set CLOUD_CONFIG=!USER_INPUT! -echo !CLOUD_CONFIG! -goto create_collection - -:create_collection "%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ - org.apache.solr.util.SolrCLI create_collection -name !CLOUD_COLLECTION! -shards !CLOUD_NUM_SHARDS! -replicationFactor !CLOUD_REPFACT! ^ - -confdir !CLOUD_CONFIG! -configsetsDir "%SOLR_SERVER_DIR%\solr\configsets" -zkHost %zk_host% + org.apache.solr.util.SolrCLI run_example -script "%SDIR%\solr.cmd" -e %EXAMPLE% -d "%SOLR_SERVER_DIR%" -urlScheme !SOLR_URL_SCHEME! !PASS_TO_RUN_EXAMPLE! -@echo. -echo Enabling auto soft-commits with maxTime 3 secs using the Config API -"%JAVA%" %SOLR_SSL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" -Dlog4j.configuration="file:%DEFAULT_SERVER_DIR%\scripts\cloud-scripts\log4j.properties" ^ - -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ - org.apache.solr.util.SolrCLI config -collection !CLOUD_COLLECTION! -property updateHandler.autoSoftCommit.maxTime -value 3000 -zkHost %zk_host% - -echo. -echo SolrCloud example is running, please visit !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%NODE1_PORT%/solr" -echo. - -REM End of interactive cloud example +REM End of run_example goto done - :get_info REM Find all Java processes, correlate with those listening on a port REM and then try to contact via that port using the status tool @@ -1383,10 +1174,8 @@ goto done exit /b 1 :done - ENDLOCAL - -GOTO :eof +exit /b 0 REM Tests what Java we have and sets some global variables :resolve_java_info diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 1495e46911d..fbaefa8ea7c 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -31,6 +31,7 @@ + diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java index 9482ea3c78b..8cfe5c7d1e4 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java @@ -20,11 +20,15 @@ package org.apache.solr.util; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.PrintStream; import java.net.ConnectException; +import java.net.Socket; import java.net.SocketException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; @@ -32,8 +36,11 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Scanner; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -44,6 +51,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.exec.DefaultExecuteResultHandler; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.Executor; +import org.apache.commons.exec.OS; +import org.apache.commons.exec.environment.EnvironmentUtils; import org.apache.commons.io.FileUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -99,23 +111,61 @@ public class SolrCLI { Option[] getOptions(); int runTool(CommandLine cli) throws Exception; } + + public static abstract class ToolBase implements Tool { + protected PrintStream stdout; + protected boolean verbose = false; + + protected ToolBase() { + this(System.out); + } + + protected ToolBase(PrintStream stdout) { + this.stdout = stdout; + } + + protected void echo(final String msg) { + stdout.println(msg); + } + + public int runTool(CommandLine cli) throws Exception { + verbose = cli.hasOption("verbose"); + + int toolExitStatus = 0; + try { + runImpl(cli); + } catch (Exception exc) { + // since this is a CLI, spare the user the stacktrace + String excMsg = exc.getMessage(); + if (excMsg != null) { + System.err.println("\nERROR: "+excMsg+"\n"); + toolExitStatus = 1; + } else { + throw exc; + } + } + return toolExitStatus; + } + + protected abstract void runImpl(CommandLine cli) throws Exception; + } /** * Helps build SolrCloud aware tools by initializing a CloudSolrClient * instance before running the tool. */ - public static abstract class SolrCloudTool implements Tool { - + public static abstract class SolrCloudTool extends ToolBase { + + protected SolrCloudTool(PrintStream stdout) { super(stdout); } + public Option[] getOptions() { return cloudOptions; } - public int runTool(CommandLine cli) throws Exception { - + protected void runImpl(CommandLine cli) throws Exception { String zkHost = cli.getOptionValue("zkHost", ZK_HOST); log.debug("Connecting to Solr cluster: " + zkHost); - int exitStatus = 0; try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { String collection = cli.getOptionValue("collection"); @@ -123,25 +173,14 @@ public class SolrCLI { cloudSolrClient.setDefaultCollection(collection); cloudSolrClient.connect(); - exitStatus = runCloudTool(cloudSolrClient, cli); - } catch (Exception exc) { - // since this is a CLI, spare the user the stacktrace - String excMsg = exc.getMessage(); - if (excMsg != null) { - System.err.println("\nERROR: "+excMsg+"\n"); - exitStatus = 1; - } else { - throw exc; - } + runCloudTool(cloudSolrClient, cli); } - - return exitStatus; } /** * Runs a SolrCloud tool with CloudSolrClient initialized */ - protected abstract int runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) + protected abstract void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception; } @@ -163,7 +202,17 @@ public class SolrCLI { .isRequired(false) .withDescription("Name of collection; no default") .create("collection") - }; + }; + + private static void exit(int exitStatus) { + // TODO: determine if we're running in a test and don't exit + try { + System.exit(exitStatus); + } catch (java.lang.SecurityException secExc) { + if (exitStatus != 0) + throw new RuntimeException("SolrCLI failed to exit with status "+exitStatus); + } + } /** * Runs a tool. @@ -173,7 +222,7 @@ public class SolrCLI { System.err.println("Invalid command-line args! Must pass the name of a tool to run.\n" + "Supported tools:\n"); displayToolOptions(System.err); - System.exit(1); + exit(1); } String configurerClassName = System.getProperty("solr.authentication.httpclient.configurer"); @@ -191,14 +240,27 @@ public class SolrCLI { // Determine the tool String toolType = args[0].trim().toLowerCase(Locale.ROOT); Tool tool = newTool(toolType); - - String[] toolArgs = new String[args.length - 1]; - System.arraycopy(args, 1, toolArgs, 0, toolArgs.length); - + + // the parser doesn't like -D props + List toolArgList = new ArrayList(); + List dashDList = new ArrayList(); + for (int a=1; a < args.length; a++) { + String arg = args[a]; + if (arg.startsWith("-D")) { + dashDList.add(arg); + } else { + toolArgList.add(arg); + } + } + String[] toolArgs = toolArgList.toArray(new String[0]); + // process command-line args to configure this application CommandLine cli = processCommandLineArgs(joinCommonAndToolOptions(tool.getOptions()), toolArgs); + List argList = cli.getArgList(); + argList.addAll(dashDList); + // for SSL support, try to accommodate relative paths set for SSL store props String solrInstallDir = System.getProperty("solr.install.dir"); if (solrInstallDir != null) { @@ -207,7 +269,7 @@ public class SolrCLI { } // run the tool - System.exit(tool.runTool(cli)); + exit(tool.runTool(cli)); } protected static void checkSslStoreSysProp(String solrInstallDir, String key) { @@ -255,6 +317,8 @@ public class SolrCLI { return new DeleteTool(); else if ("config".equals(toolType)) return new ConfigTool(); + else if ("run_example".equals(toolType)) + return new RunExampleTool(); // If you add a built-in tool to this class, add it here to avoid // classpath scanning @@ -278,6 +342,7 @@ public class SolrCLI { formatter.printHelp("create", getToolOptions(new CreateTool())); formatter.printHelp("delete", getToolOptions(new DeleteTool())); formatter.printHelp("config", getToolOptions(new ConfigTool())); + formatter.printHelp("run_example", getToolOptions(new RunExampleTool())); List> toolClasses = findToolClassesInPackage("org.apache.solr.util"); for (Class next : toolClasses) { @@ -288,8 +353,8 @@ public class SolrCLI { private static Options getToolOptions(Tool tool) { Options options = new Options(); - options.addOption("h", "help", false, "Print this message"); - options.addOption("v", "verbose", false, "Generate verbose log messages"); + options.addOption("help", false, "Print this message"); + options.addOption("verbose", false, "Generate verbose log messages"); Option[] toolOpts = joinCommonAndToolOptions(tool.getOptions()); for (int i = 0; i < toolOpts.length; i++) options.addOption(toolOpts[i]); @@ -322,8 +387,8 @@ public class SolrCLI { public static CommandLine processCommandLineArgs(Option[] customOptions, String[] args) { Options options = new Options(); - options.addOption("h", "help", false, "Print this message"); - options.addOption("v", "verbose", false, "Generate verbose log messages"); + options.addOption("help", false, "Print this message"); + options.addOption("verbose", false, "Generate verbose log messages"); if (customOptions != null) { for (int i = 0; i < customOptions.length; i++) @@ -337,7 +402,7 @@ public class SolrCLI { boolean hasHelpArg = false; if (args != null && args.length > 0) { for (int z = 0; z < args.length; z++) { - if ("-h".equals(args[z]) || "-help".equals(args[z])) { + if ("--help".equals(args[z]) || "-help".equals(args[z])) { hasHelpArg = true; break; } @@ -349,13 +414,13 @@ public class SolrCLI { } HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(SolrCLI.class.getName(), options); - System.exit(1); + exit(1); } if (cli.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(SolrCLI.class.getName(), options); - System.exit(0); + exit(0); } return cli; @@ -384,7 +449,7 @@ public class SolrCLI { } } catch (Exception e) { // safe to squelch this as it's just looking for tools to run - //e.printStackTrace(); + log.debug("Failed to find Tool impl classes in "+packageName+" due to: "+e); } return toolClasses; } @@ -442,6 +507,16 @@ public class SolrCLI { } } + public static final String JSON_CONTENT_TYPE = "application/json"; + + public static NamedList postJsonToSolr(SolrClient solrClient, String updatePath, String jsonBody) throws Exception { + ContentStreamBase.StringStream contentStream = new ContentStreamBase.StringStream(jsonBody); + contentStream.setContentType(JSON_CONTENT_TYPE); + ContentStreamUpdateRequest req = new ContentStreamUpdateRequest(updatePath); + req.addContentStream(contentStream); + return solrClient.request(req); + } + /** * Useful when a tool just needs to send one request to Solr. */ @@ -449,7 +524,7 @@ public class SolrCLI { Map json = null; CloseableHttpClient httpClient = getHttpClient(); try { - json = getJson(httpClient, getUrl, 2); + json = getJson(httpClient, getUrl, 2, true); } finally { closeHttpClient(httpClient); } @@ -459,21 +534,22 @@ public class SolrCLI { /** * Utility function for sending HTTP GET request to Solr with built-in retry support. */ - public static Map getJson(HttpClient httpClient, String getUrl, int attempts) throws Exception { + public static Map getJson(HttpClient httpClient, String getUrl, int attempts, boolean isFirstAttempt) throws Exception { Map json = null; if (attempts >= 1) { try { json = getJson(httpClient, getUrl); } catch (Exception exc) { if (--attempts > 0 && checkCommunicationError(exc)) { - log.warn("Request to "+getUrl+" failed due to: "+exc.getMessage()+ - ", sleeping for 5 seconds before re-trying the request ..."); + if (!isFirstAttempt) // only show the log warning after the second attempt fails + log.warn("Request to "+getUrl+" failed due to: "+exc.getMessage()+ + ", sleeping for 5 seconds before re-trying the request ..."); try { Thread.sleep(5000); } catch (InterruptedException ie) { Thread.interrupted(); } // retry using recursion with one-less attempt available - json = getJson(httpClient, getUrl, attempts); + json = getJson(httpClient, getUrl, attempts, false); } else { // no more attempts or error is not retry-able throw exc; @@ -530,6 +606,17 @@ public class SolrCLI { if (errMsg == null) errMsg = String.valueOf(json); throw new SolrServerException(errMsg); + } else { + // make sure no "failure" object in there either + Object failureObj = json.get("failure"); + if (failureObj != null) { + if (failureObj instanceof Map) { + Object err = ((Map)failureObj).get(""); + if (err != null) + throw new SolrServerException(err.toString()); + } + throw new SolrServerException(failureObj.toString()); + } } return json; } @@ -620,15 +707,16 @@ public class SolrCLI { /** * Get the status of a Solr server. */ - public static class StatusTool implements Tool { + public static class StatusTool extends ToolBase { + + public StatusTool() { this(System.out); } + public StatusTool(PrintStream stdout) { super(stdout); } - @Override public String getName() { return "status"; } @SuppressWarnings("static-access") - @Override public Option[] getOptions() { return new Option[] { OptionBuilder @@ -636,46 +724,78 @@ public class SolrCLI { .hasArg() .isRequired(false) .withDescription("Address of the Solr Web application, defaults to: "+DEFAULT_SOLR_URL) - .create("solr") + .create("solr"), + OptionBuilder + .withArgName("SECS") + .hasArg() + .isRequired(false) + .withDescription("Wait up to the specified number of seconds to see Solr running.") + .create("maxWaitSecs") }; } - @Override - public int runTool(CommandLine cli) throws Exception { + protected void runImpl(CommandLine cli) throws Exception { + int maxWaitSecs = Integer.parseInt(cli.getOptionValue("maxWaitSecs", "0")); String solrUrl = cli.getOptionValue("solr", DEFAULT_SOLR_URL); + if (maxWaitSecs > 0) { + int solrPort = (new URL(solrUrl)).getPort(); + echo("Waiting up to "+maxWaitSecs+" to see Solr running on port "+solrPort); + try { + waitToSeeSolrUp(solrUrl, maxWaitSecs); + echo("Started Solr server on port "+solrPort+". Happy searching!"); + } catch (TimeoutException timeout) { + throw new Exception("Solr at "+solrUrl+" did not come online within "+maxWaitSecs+" seconds!"); + } + } else { + try { + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(getStatus(solrUrl)); + echo(arr.toString()); + } catch (Exception exc) { + if (checkCommunicationError(exc)) { + // this is not actually an error from the tool as it's ok if Solr is not online. + System.err.println("Solr at "+solrUrl+" not online."); + } else { + throw new Exception("Failed to get system information from " + solrUrl + " due to: "+exc); + } + } + } + } + + public Map waitToSeeSolrUp(String solrUrl, int maxWaitSecs) throws Exception { + long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(maxWaitSecs, TimeUnit.SECONDS); + while (System.nanoTime() < timeout) { + try { + return getStatus(solrUrl); + } catch (Exception exc) { + try { + Thread.sleep(2000L); + } catch (InterruptedException interrupted) { + timeout = 0; // stop looping + } + } + } + throw new TimeoutException("Did not see Solr at "+solrUrl+" come online within "+maxWaitSecs); + } + + public Map getStatus(String solrUrl) throws Exception { + Map status = null; + if (!solrUrl.endsWith("/")) solrUrl += "/"; - int exitCode = 0; String systemInfoUrl = solrUrl+"admin/info/system"; CloseableHttpClient httpClient = getHttpClient(); try { // hit Solr to get system info - Map systemInfo = getJson(httpClient, systemInfoUrl, 2); - + Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); // convert raw JSON into user-friendly output - Map status = - reportStatus(solrUrl, systemInfo, httpClient); - - // pretty-print the status to stdout - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(status); - System.out.println(arr.toString()); - - } catch (Exception exc) { - if (checkCommunicationError(exc)) { - // this is not actually an error from the tool as it's ok if Solr is not online. - System.err.println("Solr at "+solrUrl+" not online."); - } else { - System.err.print("Failed to get system information from "+solrUrl+" due to: "); - exc.printStackTrace(System.err); - exitCode = 1; - } + status = reportStatus(solrUrl, systemInfo, httpClient); } finally { closeHttpClient(httpClient); } - return exitCode; + return status; } public Map reportStatus(String solrUrl, Map info, HttpClient httpClient) @@ -713,7 +833,7 @@ public class SolrCLI { cloudStatus.put("ZooKeeper", (zkHost != null) ? zkHost : "?"); String clusterStatusUrl = solrUrl+"admin/collections?action=CLUSTERSTATUS"; - Map json = getJson(httpClient, clusterStatusUrl, 2); + Map json = getJson(httpClient, clusterStatusUrl, 2, true); List liveNodes = asList("/cluster/live_nodes", json); cloudStatus.put("liveNodes", String.valueOf(liveNodes.size())); @@ -729,15 +849,16 @@ public class SolrCLI { /** * Used to send an arbitrary HTTP request to a Solr API endpoint. */ - public static class ApiTool implements Tool { + public static class ApiTool extends ToolBase { + + public ApiTool() { this(System.out); } + public ApiTool(PrintStream stdout) { super(stdout); } - @Override public String getName() { return "api"; } @SuppressWarnings("static-access") - @Override public Option[] getOptions() { return new Option[] { OptionBuilder @@ -749,8 +870,7 @@ public class SolrCLI { }; } - @Override - public int runTool(CommandLine cli) throws Exception { + protected void runImpl(CommandLine cli) throws Exception { String getUrl = cli.getOptionValue("get"); if (getUrl != null) { Map json = getJson(getUrl); @@ -758,11 +878,9 @@ public class SolrCLI { // pretty-print the response to stdout CharArr arr = new CharArr(); new JSONWriter(arr, 2).write(json); - System.out.println(arr.toString()); + echo(arr.toString()); } - - return 0; - } + } } // end ApiTool class private static final String DEFAULT_CONFIG_SET = "data_driven_schema_configs"; @@ -918,14 +1036,17 @@ public class SolrCLI { * Requests health information about a specific collection in SolrCloud. */ public static class HealthcheckTool extends SolrCloudTool { - + + public HealthcheckTool() { this(System.out); } + public HealthcheckTool(PrintStream stdout) { super(stdout); } + @Override public String getName() { return "healthcheck"; } @Override - protected int runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { + protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { String collection = cli.getOptionValue("collection"); if (collection == null) @@ -995,7 +1116,7 @@ public class SolrCLI { int lastSlash = solrUrl.lastIndexOf('/'); String systemInfoUrl = solrUrl.substring(0,lastSlash)+"/admin/info/system"; - Map info = getJson(solr.getHttpClient(), systemInfoUrl, 2); + Map info = getJson(solr.getHttpClient(), systemInfoUrl, 2, true); uptime = uptime(asLong("/jvm/jmx/upTimeMS", info)); String usedMemory = asString("/jvm/memory/used", info); String totalMemory = asString("/jvm/memory/total", info); @@ -1038,19 +1159,17 @@ public class SolrCLI { CharArr arr = new CharArr(); new JSONWriter(arr, 2).write(report); - System.out.println(arr.toString()); - - return 0; + echo(arr.toString()); } } // end HealthcheckTool private static final Option[] CREATE_COLLECTION_OPTIONS = new Option[] { - OptionBuilder - .withArgName("HOST") - .hasArg() - .isRequired(false) - .withDescription("Address of the Zookeeper ensemble; defaults to: "+ZK_HOST) - .create("zkHost"), + OptionBuilder + .withArgName("HOST") + .hasArg() + .isRequired(false) + .withDescription("Address of the Zookeeper ensemble; defaults to: " + ZK_HOST) + .create("zkHost"), OptionBuilder .withArgName("HOST") .hasArg() @@ -1146,7 +1265,7 @@ public class SolrCLI { CloseableHttpClient httpClient = getHttpClient(); try { // hit Solr to get system info - Map systemInfo = getJson(httpClient, systemInfoUrl, 2); + Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); // convert raw JSON into user-friendly output StatusTool statusTool = new StatusTool(); @@ -1178,58 +1297,69 @@ public class SolrCLI { return exists; } + public static boolean safeCheckCoreExists(String coreStatusUrl, String coreName) { + boolean exists = false; + try { + Map existsCheckResult = getJson(coreStatusUrl); + Map status = (Map)existsCheckResult.get("status"); + Map coreStatus = (Map)status.get(coreName); + exists = coreStatus != null && coreStatus.containsKey(NAME); + } catch (Exception exc) { + // just ignore it since we're only interested in a positive result here + } + return exists; + } + /** * Supports create_collection command in the bin/solr script. */ - public static class CreateCollectionTool implements Tool { + public static class CreateCollectionTool extends ToolBase { + + public CreateCollectionTool() { + this(System.out); + } + + public CreateCollectionTool(PrintStream stdout) { + super(stdout); + } - @Override public String getName() { return "create_collection"; } @SuppressWarnings("static-access") - @Override public Option[] getOptions() { return CREATE_COLLECTION_OPTIONS; } - public int runTool(CommandLine cli) throws Exception { + protected void runImpl(CommandLine cli) throws Exception { String zkHost = getZkHost(cli); if (zkHost == null) { - System.err.println("\nERROR: Solr at "+cli.getOptionValue("solrUrl")+ + throw new IllegalStateException("Solr at "+cli.getOptionValue("solrUrl")+ " is running in standalone server mode, please use the create_core command instead;\n" + "create_collection can only be used when running in SolrCloud mode.\n"); - return 1; } - int toolExitStatus = 0; - try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { - System.out.println("Connecting to ZooKeeper at " + zkHost); + echo("\nConnecting to ZooKeeper at " + zkHost+" ..."); cloudSolrClient.connect(); - toolExitStatus = runCloudTool(cloudSolrClient, cli); - } catch (Exception exc) { - // since this is a CLI, spare the user the stacktrace - String excMsg = exc.getMessage(); - if (excMsg != null) { - System.err.println("\nERROR: "+excMsg+"\n"); - toolExitStatus = 1; - } else { - throw exc; - } + runCloudTool(cloudSolrClient, cli); } - - return toolExitStatus; } - protected int runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { + protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { + Set liveNodes = cloudSolrClient.getZkStateReader().getClusterState().getLiveNodes(); if (liveNodes.isEmpty()) throw new IllegalStateException("No live nodes found! Cannot create a collection until " + "there is at least 1 live node in the cluster."); - String firstLiveNode = liveNodes.iterator().next(); + + String baseUrl = cli.getOptionValue("solrUrl"); + if (baseUrl == null) { + String firstLiveNode = liveNodes.iterator().next(); + baseUrl = cloudSolrClient.getZkStateReader().getBaseUrlForNodeName(firstLiveNode); + } String collectionName = cli.getOptionValue(NAME); @@ -1248,12 +1378,12 @@ public class SolrCLI { String confname = cli.getOptionValue("confname", collectionName); boolean configExistsInZk = - cloudSolrClient.getZkStateReader().getZkClient().exists("/configs/"+confname, true); + cloudSolrClient.getZkStateReader().getZkClient().exists("/configs/" + confname, true); if (".system".equals(collectionName)) { //do nothing } else if (configExistsInZk) { - System.out.println("Re-using existing configuration directory "+confname); + echo("Re-using existing configuration directory "+confname); } else { String configSet = cli.getOptionValue("confdir", DEFAULT_CONFIG_SET); File configSetDir = null; @@ -1280,27 +1410,23 @@ public class SolrCLI { if ((new File(configSetDir, "solrconfig.xml")).isFile()) { confDir = configSetDir; } else { - System.err.println("Specified configuration directory "+configSetDir.getAbsolutePath()+ + throw new IllegalArgumentException("Specified configuration directory "+configSetDir.getAbsolutePath()+ " is invalid;\nit should contain either conf sub-directory or solrconfig.xml"); - return 1; } } // test to see if that config exists in ZK - System.out.println("Uploading "+confDir.getAbsolutePath()+ + echo("Uploading "+confDir.getAbsolutePath()+ " for config "+confname+" to ZooKeeper at "+cloudSolrClient.getZkHost()); cloudSolrClient.uploadConfig(confDir.toPath(), confname); } - String baseUrl = cloudSolrClient.getZkStateReader().getBaseUrlForNodeName(firstLiveNode); - // since creating a collection is a heavy-weight operation, check for existence first String collectionListUrl = baseUrl+"/admin/collections?action=list"; if (safeCheckCollectionExists(collectionListUrl, collectionName)) { - System.err.println("\nCollection '"+collectionName+"' already exists!"); - System.err.println("\nChecked collection existence using Collections API command:\n"+collectionListUrl); - System.err.println(); - return 1; + throw new IllegalStateException("\nCollection '"+collectionName+ + "' already exists!\nChecked collection existence using Collections API command:\n"+ + collectionListUrl); } // doesn't seem to exist ... try to create @@ -1314,7 +1440,7 @@ public class SolrCLI { maxShardsPerNode, confname); - System.out.println("\nCreating new collection '"+collectionName+"' using command:\n"+createCollectionUrl+"\n"); + echo("\nCreating new collection '"+collectionName+"' using command:\n"+createCollectionUrl+"\n"); Map json = null; try { @@ -1322,21 +1448,16 @@ public class SolrCLI { } catch (SolrServerException sse) { // check if already exists if (safeCheckCollectionExists(collectionListUrl, collectionName)) { - System.err.println("Collection '"+collectionName+"' already exists!"); - System.err.println("\nChecked collection existence using Collections API command:\n"+collectionListUrl); + throw new IllegalArgumentException("Collection '"+collectionName+ + "' already exists!\nChecked collection existence using Collections API command:\n"+collectionListUrl); } else { - System.err.println("Failed to create collection '"+collectionName+"' due to: "+sse.getMessage()); + throw new Exception("Failed to create collection '"+collectionName+"' due to: "+sse.getMessage()); } - System.err.println(); - return 1; } CharArr arr = new CharArr(); new JSONWriter(arr, 2).write(json); - System.out.println(arr.toString()); - System.out.println(); - - return 0; + echo(arr.toString()); } protected int optionAsInt(CommandLine cli, String option, int defaultVal) { @@ -1344,15 +1465,16 @@ public class SolrCLI { } } // end CreateCollectionTool class - public static class CreateCoreTool implements Tool { + public static class CreateCoreTool extends ToolBase { + + public CreateCoreTool() { this(System.out); } + public CreateCoreTool(PrintStream stdout) { super(stdout); } - @Override public String getName() { return "create_core"; } @SuppressWarnings("static-access") - @Override public Option[] getOptions() { return new Option[] { OptionBuilder @@ -1382,8 +1504,7 @@ public class SolrCLI { }; } - @Override - public int runTool(CommandLine cli) throws Exception { + protected void runImpl(CommandLine cli) throws Exception { String solrUrl = cli.getOptionValue("solrUrl", DEFAULT_SOLR_URL); if (!solrUrl.endsWith("/")) @@ -1412,11 +1533,10 @@ public class SolrCLI { CloseableHttpClient httpClient = getHttpClient(); String solrHome = null; try { - Map systemInfo = getJson(httpClient, systemInfoUrl, 2); + Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); if ("solrcloud".equals(systemInfo.get("mode"))) { - System.err.println("\nERROR: Solr at "+solrUrl+ - " is running in SolrCloud mode, please use create_collection command instead.\n"); - return 1; + throw new IllegalStateException("Solr at "+solrUrl+ + " is running in SolrCloud mode, please use create_collection command instead."); } // convert raw JSON into user-friendly output @@ -1430,10 +1550,8 @@ public class SolrCLI { String coreStatusUrl = solrUrl+"admin/cores?action=STATUS&core="+coreName; if (safeCheckCoreExists(coreStatusUrl, coreName)) { - System.err.println("\nCore '"+coreName+"' already exists!"); - System.err.println("\nChecked core existence using Core API command:\n"+coreStatusUrl); - System.err.println(); - return 1; + throw new IllegalArgumentException("\nCore '"+coreName+ + "' already exists!\nChecked core existence using Core API command:\n"+coreStatusUrl); } File coreInstanceDir = new File(solrHome, coreName); @@ -1451,11 +1569,10 @@ public class SolrCLI { if ((new File(configSetDir, "solrconfig.xml")).isFile()) { FileUtils.copyDirectory(configSetDir, new File(coreInstanceDir, "conf")); } else { - System.err.println("\n"+configSetDir.getAbsolutePath()+" doesn't contain a conf subdirectory or solrconfig.xml\n"); - return 1; + throw new IllegalArgumentException("\n"+configSetDir.getAbsolutePath()+" doesn't contain a conf subdirectory or solrconfig.xml\n"); } } - System.out.println("\nSetup new core instance directory:\n"+coreInstanceDir.getAbsolutePath()); + echo("\nSetup new core instance directory:\n" + coreInstanceDir.getAbsolutePath()); } String createCoreUrl = @@ -1465,7 +1582,7 @@ public class SolrCLI { coreName, coreName); - System.out.println("\nCreating new core '"+coreName+"' using command:\n"+createCoreUrl+"\n"); + echo("\nCreating new core '" + coreName + "' using command:\n" + createCoreUrl + "\n"); Map json = null; try { @@ -1474,52 +1591,34 @@ public class SolrCLI { // mostly likely the core already exists ... if (safeCheckCoreExists(coreStatusUrl, coreName)) { // core already exists - System.err.println("Core '"+coreName+"' already exists!"); - System.err.println("\nChecked core existence using Core API command:\n"+coreStatusUrl); + throw new IllegalArgumentException("Core '"+coreName+"' already exists!\nChecked core existence using Core API command:\n"+coreStatusUrl); } else { - System.err.println("Failed to create core '"+coreName+"' due to: "+sse.getMessage()); + throw sse; } - System.err.println(); - return 1; } CharArr arr = new CharArr(); new JSONWriter(arr, 2).write(json); - System.out.println(arr.toString()); - System.out.println(); - - return 0; - } - - protected boolean safeCheckCoreExists(String coreStatusUrl, String coreName) { - boolean exists = false; - try { - Map existsCheckResult = getJson(coreStatusUrl); - Map status = (Map)existsCheckResult.get("status"); - Map coreStatus = (Map)status.get(coreName); - exists = coreStatus != null && coreStatus.containsKey(NAME); - } catch (Exception exc) { - // just ignore it since we're only interested in a positive result here - } - return exists; + echo(arr.toString()); + echo("\n"); } } // end CreateCoreTool class - public static class CreateTool implements Tool { + public static class CreateTool extends ToolBase { + + public CreateTool() { this(System.out); } + public CreateTool(PrintStream stdout) { super(stdout); } - @Override public String getName() { return "create"; } @SuppressWarnings("static-access") - @Override public Option[] getOptions() { return CREATE_COLLECTION_OPTIONS; } - @Override - public int runTool(CommandLine cli) throws Exception { + protected void runImpl(CommandLine cli) throws Exception { String solrUrl = cli.getOptionValue("solrUrl", DEFAULT_SOLR_URL); if (!solrUrl.endsWith("/")) @@ -1528,38 +1627,32 @@ public class SolrCLI { String systemInfoUrl = solrUrl+"admin/info/system"; CloseableHttpClient httpClient = getHttpClient(); - int result = -1; Tool tool = null; try { - Map systemInfo = getJson(httpClient, systemInfoUrl, 2); + Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); if ("solrcloud".equals(systemInfo.get("mode"))) { - tool = new CreateCollectionTool(); + tool = new CreateCollectionTool(stdout); } else { - tool = new CreateCoreTool(); + tool = new CreateCoreTool(stdout); } - result = tool.runTool(cli); - } catch (Exception exc) { - System.err.println("ERROR: create failed due to: "+exc.getMessage()); - System.err.println(); - result = 1; + tool.runTool(cli); } finally { closeHttpClient(httpClient); } - - return result; } } // end CreateTool class - public static class DeleteTool implements Tool { + public static class DeleteTool extends ToolBase { + + public DeleteTool() { this(System.out); } + public DeleteTool(PrintStream stdout) { super(stdout); } - @Override public String getName() { return "delete"; } @SuppressWarnings("static-access") - @Override public Option[] getOptions() { return new Option[]{ OptionBuilder @@ -1593,8 +1686,7 @@ public class SolrCLI { }; } - @Override - public int runTool(CommandLine cli) throws Exception { + protected void runImpl(CommandLine cli) throws Exception { String solrUrl = cli.getOptionValue("solrUrl", DEFAULT_SOLR_URL); if (!solrUrl.endsWith("/")) @@ -1602,60 +1694,39 @@ public class SolrCLI { String systemInfoUrl = solrUrl+"admin/info/system"; CloseableHttpClient httpClient = getHttpClient(); - - int result = 0; try { - Map systemInfo = getJson(httpClient, systemInfoUrl, 2); + Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); if ("solrcloud".equals(systemInfo.get("mode"))) { - result = deleteCollection(cli); + deleteCollection(cli); } else { - result = deleteCore(cli, httpClient, solrUrl); + deleteCore(cli, httpClient, solrUrl); } } finally { closeHttpClient(httpClient); } - - return result; } - protected int deleteCollection(CommandLine cli) throws Exception { - + protected void deleteCollection(CommandLine cli) throws Exception { String zkHost = getZkHost(cli); - - int toolExitStatus = 0; try (CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost)) { - System.out.println("Connecting to ZooKeeper at " + zkHost); + echo("Connecting to ZooKeeper at " + zkHost); cloudSolrClient.connect(); - toolExitStatus = deleteCollection(cloudSolrClient, cli); - } catch (Exception exc) { - // since this is a CLI, spare the user the stacktrace - String excMsg = exc.getMessage(); - if (excMsg != null) { - System.err.println("\nERROR: "+excMsg+"\n"); - toolExitStatus = 1; - } else { - throw exc; - } + deleteCollection(cloudSolrClient, cli); } - - return toolExitStatus; } - protected int deleteCollection(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { + protected void deleteCollection(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { Set liveNodes = cloudSolrClient.getZkStateReader().getClusterState().getLiveNodes(); if (liveNodes.isEmpty()) throw new IllegalStateException("No live nodes found! Cannot delete a collection until " + "there is at least 1 live node in the cluster."); + String firstLiveNode = liveNodes.iterator().next(); ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); String baseUrl = zkStateReader.getBaseUrlForNodeName(firstLiveNode); - String collectionName = cli.getOptionValue(NAME); - if (!zkStateReader.getClusterState().hasCollection(collectionName)) { - System.err.println("\nERROR: Collection "+collectionName+" not found!"); - System.err.println(); - return 1; + throw new IllegalArgumentException("Collection "+collectionName+" not found!"); } String configName = zkStateReader.readConfigName(collectionName); @@ -1692,15 +1763,13 @@ public class SolrCLI { baseUrl, collectionName); - System.out.println("\nDeleting collection '"+collectionName+"' using command:\n"+deleteCollectionUrl+"\n"); + echo("\nDeleting collection '" + collectionName + "' using command:\n" + deleteCollectionUrl + "\n"); Map json = null; try { json = getJson(deleteCollectionUrl); } catch (SolrServerException sse) { - System.err.println("Failed to delete collection '"+collectionName+"' due to: "+sse.getMessage()); - System.err.println(); - return 1; + throw new Exception("Failed to delete collection '"+collectionName+"' due to: "+sse.getMessage()); } if (deleteConfig) { @@ -1708,7 +1777,7 @@ public class SolrCLI { try { zkStateReader.getZkClient().clean(configZnode); } catch (Exception exc) { - System.err.println("\nERROR: Failed to delete configuration directory "+configZnode+" in ZooKeeper due to: "+ + System.err.println("\nWARNING: Failed to delete configuration directory "+configZnode+" in ZooKeeper due to: "+ exc.getMessage()+"\nYou'll need to manually delete this znode using the zkcli script."); } } @@ -1716,16 +1785,12 @@ public class SolrCLI { if (json != null) { CharArr arr = new CharArr(); new JSONWriter(arr, 2).write(json); - System.out.println(arr.toString()); - System.out.println(); + echo(arr.toString()); + echo("\n"); } - - return 0; } - protected int deleteCore(CommandLine cli, CloseableHttpClient httpClient, String solrUrl) throws Exception { - - int status = 0; + protected void deleteCore(CommandLine cli, CloseableHttpClient httpClient, String solrUrl) throws Exception { String coreName = cli.getOptionValue(NAME); String deleteCoreUrl = String.format(Locale.ROOT, @@ -1733,35 +1798,32 @@ public class SolrCLI { solrUrl, coreName); - System.out.println("\nDeleting core '"+coreName+"' using command:\n"+deleteCoreUrl+"\n"); + echo("\nDeleting core '" + coreName + "' using command:\n" + deleteCoreUrl + "\n"); Map json = null; try { json = getJson(deleteCoreUrl); } catch (SolrServerException sse) { - System.err.println("Failed to delete core '"+coreName+"' due to: "+sse.getMessage()); - System.err.println(); - status = 1; + throw new Exception("Failed to delete core '"+coreName+"' due to: "+sse.getMessage()); } if (json != null) { CharArr arr = new CharArr(); new JSONWriter(arr, 2).write(json); - System.out.println(arr.toString()); - System.out.println(); + echo(arr.toString()); + echo("\n"); } - - return status; } - } // end DeleteTool class /** * Sends a POST to the Config API to perform a specified action. */ - public static class ConfigTool implements Tool { + public static class ConfigTool extends ToolBase { + + public ConfigTool() { this(System.out); } + public ConfigTool(PrintStream stdout) { super(stdout); } - @Override public String getName() { return "config"; } @@ -1769,7 +1831,7 @@ public class SolrCLI { @SuppressWarnings("static-access") @Override public Option[] getOptions() { - return new Option[] { + Option[] configOptions = new Option[] { OptionBuilder .withArgName("ACTION") .hasArg() @@ -1788,18 +1850,6 @@ public class SolrCLI { .isRequired(false) .withDescription("Set the property to this value; accepts JSON objects and strings") .create("value"), - OptionBuilder - .withArgName("COLL") - .hasArg() - .isRequired(false) - .withDescription("Collection; defaults to gettingstarted") - .create("collection"), - OptionBuilder - .withArgName("HOST") - .hasArg() - .isRequired(false) - .withDescription("Address of the Zookeeper ensemble") - .create("zkHost"), OptionBuilder .withArgName("HOST") .hasArg() @@ -1807,10 +1857,10 @@ public class SolrCLI { .withDescription("Base Solr URL, which can be used to determine the zkHost if that's not known") .create("solrUrl") }; + return joinOptions(configOptions, cloudOptions); } - @Override - public int runTool(CommandLine cli) throws Exception { + protected void runImpl(CommandLine cli) throws Exception { String solrUrl = resolveSolrUrl(cli); String action = cli.getOptionValue("action", "set-property"); String collection = cli.getOptionValue("collection", "gettingstarted"); @@ -1832,38 +1882,797 @@ public class SolrCLI { String updatePath = "/"+collection+"/config"; - System.out.println("\nPOSTing request to Config API: "+solrUrl+updatePath); - System.out.println(jsonBody); - System.out.println(); + echo("\nPOSTing request to Config API: " + solrUrl + updatePath); + echo(jsonBody); - int exitStatus = 0; try (SolrClient solrClient = new HttpSolrClient(solrUrl)) { NamedList result = postJsonToSolr(solrClient, updatePath, jsonBody); Integer statusCode = (Integer)((NamedList)result.get("responseHeader")).get("status"); if (statusCode == 0) { if (value != null) { - System.out.println("Successfully "+action+" "+property+" to "+value); + echo("Successfully " + action + " " + property + " to " + value); } else { - System.out.println("Successfully "+action+" "+property); + echo("Successfully " + action + " " + property); } } else { - String errMsg = "Failed to "+action+" property due to:\n"+result; - System.err.println("\nERROR: "+errMsg+"\n"); - exitStatus = 1; + throw new Exception("Failed to "+action+" property due to:\n"+result); } } - return exitStatus; } } // end ConfigTool class - public static final String JSON_CONTENT_TYPE = "application/json"; + /** + * Supports an interactive session with the user to launch (or relaunch the -e cloud example) + */ + public static class RunExampleTool extends ToolBase { - public static NamedList postJsonToSolr(SolrClient solrClient, String updatePath, String jsonBody) throws Exception { - ContentStreamBase.StringStream contentStream = new ContentStreamBase.StringStream(jsonBody); - contentStream.setContentType(JSON_CONTENT_TYPE); - ContentStreamUpdateRequest req = new ContentStreamUpdateRequest(updatePath); - req.addContentStream(contentStream); - return solrClient.request(req); - } + private static final String PROMPT_FOR_NUMBER = "Please enter %s [%d]: "; + private static final String PROMPT_FOR_NUMBER_IN_RANGE = "Please enter %s between %d and %d [%d]: "; + private static final String PROMPT_NUMBER_TOO_SMALL = "%d is too small! "+PROMPT_FOR_NUMBER_IN_RANGE; + private static final String PROMPT_NUMBER_TOO_LARGE = "%d is too large! "+PROMPT_FOR_NUMBER_IN_RANGE; + + protected InputStream userInput; + protected Executor executor; + protected String script; + protected File serverDir; + protected File exampleDir; + protected String urlScheme; + + /** + * Default constructor used by the framework when running as a command-line application. + */ + public RunExampleTool() { + this(null, System.in, System.out); + } + + public RunExampleTool(Executor executor, InputStream userInput, PrintStream stdout) { + super(stdout); + this.executor = (executor != null) ? executor : new DefaultExecutor(); + this.userInput = userInput; + } + + public String getName() { + return "run_example"; + } + + @SuppressWarnings("static-access") + public Option[] getOptions() { + return new Option[] { + OptionBuilder + .isRequired(false) + .withDescription("Don't prompt for input; accept all defaults when running examples that accept user input") + .create("noprompt"), + OptionBuilder + .withArgName("NAME") + .hasArg() + .isRequired(true) + .withDescription("Name of the example to launch, one of: cloud, techproducts, dih, schemaless") + .withLongOpt("example") + .create('e'), + OptionBuilder + .withArgName("PATH") + .hasArg() + .isRequired(false) + .withDescription("Path to the bin/solr script") + .create("script"), + OptionBuilder + .withArgName("DIR") + .hasArg() + .isRequired(true) + .withDescription("Path to the Solr server directory.") + .withLongOpt("serverDir") + .create('d'), + OptionBuilder + .withArgName("DIR") + .hasArg() + .isRequired(false) + .withDescription("Path to the Solr example directory; if not provided, ${serverDir}/../example is expected to exist.") + .create("exampleDir"), + OptionBuilder + .withArgName("SCHEME") + .hasArg() + .isRequired(false) + .withDescription("Solr URL scheme: http or https, defaults to http if not specified") + .create("urlScheme"), + OptionBuilder + .withArgName("PORT") + .hasArg() + .isRequired(false) + .withDescription("Specify the port to start the Solr HTTP listener on; default is 8983") + .withLongOpt("port") + .create('p'), + OptionBuilder + .withArgName("HOSTNAME") + .hasArg() + .isRequired(false) + .withDescription("Specify the hostname for this Solr instance") + .withLongOpt("host") + .create('h'), + OptionBuilder + .withArgName("ZKHOST") + .hasArg() + .isRequired(false) + .withDescription("ZooKeeper connection string; only used when running in SolrCloud mode using -c") + .withLongOpt("zkhost") + .create('z'), + OptionBuilder + .isRequired(false) + .withDescription("Start Solr in SolrCloud mode; if -z not supplied, an embedded ZooKeeper instance is started on Solr port+1000, such as 9983 if Solr is bound to 8983") + .withLongOpt("cloud") + .create('c'), + OptionBuilder + .withArgName("MEM") + .hasArg() + .isRequired(false) + .withDescription("Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g results in: -Xms4g -Xmx4g; by default, this script sets the heap size to 512m") + .withLongOpt("memory") + .create('m'), + OptionBuilder + .withArgName("OPTS") + .hasArg() + .isRequired(false) + .withDescription("Additional options to be passed to the JVM when starting example Solr server(s)") + .withLongOpt("addlopts") + .create('a') + }; + } + + protected void runImpl(CommandLine cli) throws Exception { + this.urlScheme = cli.getOptionValue("urlScheme", "http"); + + serverDir = new File(cli.getOptionValue("serverDir")); + if (!serverDir.isDirectory()) + throw new IllegalArgumentException("Value of -serverDir option is invalid! "+ + serverDir.getAbsolutePath()+" is not a directory!"); + + script = cli.getOptionValue("script"); + if (script != null) { + if (!(new File(script)).isFile()) + throw new IllegalArgumentException("Value of -script option is invalid! "+script+" not found"); + } else { + File scriptFile = new File(serverDir.getParentFile(), "bin/solr"); + if (scriptFile.isFile()) { + script = scriptFile.getAbsolutePath(); + } else { + scriptFile = new File(serverDir.getParentFile(), "bin/solr.cmd"); + if (scriptFile.isFile()) { + script = scriptFile.getAbsolutePath(); + } else { + throw new IllegalArgumentException("Cannot locate the bin/solr script! Please pass -script to this application."); + } + } + } + + exampleDir = + (cli.hasOption("exampleDir")) ? new File(cli.getOptionValue("exampleDir")) + : new File(serverDir.getParent(), "example"); + if (!exampleDir.isDirectory()) + throw new IllegalArgumentException("Value of -exampleDir option is invalid! "+ + exampleDir.getAbsolutePath()+" is not a directory!"); + + if (verbose) { + echo("Running with\nserverDir="+serverDir.getAbsolutePath()+ + ",\nexampleDir="+exampleDir.getAbsolutePath()+"\nscript="+script); + } + + String exampleType = cli.getOptionValue("example"); + if ("cloud".equals(exampleType)) { + runCloudExample(cli); + } else if ("dih".equals(exampleType)) { + runDihExample(cli); + } else if ("techproducts".equals(exampleType) || "schemaless".equals(exampleType)) { + runExample(cli, exampleType); + } else { + throw new IllegalArgumentException("Unsupported example "+exampleType+ + "! Please choose one of: cloud, dih, schemaless, or techproducts"); + } + } + + protected void runDihExample(CommandLine cli) throws Exception { + File dihSolrHome = new File(exampleDir, "example-DIH/solr"); + if (!dihSolrHome.isDirectory()) { + dihSolrHome = new File(serverDir.getParentFile(), "example/example-DIH/solr"); + if (!dihSolrHome.isDirectory()) { + throw new Exception("example/example-DIH/solr directory not found"); + } + } + + boolean isCloudMode = cli.hasOption('c'); + String zkHost = cli.getOptionValue('z'); + int port = Integer.parseInt(cli.getOptionValue('p', "8983")); + + Map nodeStatus = startSolr(dihSolrHome, isCloudMode, cli, port, zkHost, 30); + String solrUrl = (String)nodeStatus.get("baseUrl"); + echo("\nSolr dih example launched successfully. Direct your Web browser to "+solrUrl+" to visit the Solr Admin UI"); + } + + protected void runExample(CommandLine cli, String exampleName) throws Exception { + File exDir = setupExampleDir(serverDir, exampleDir, exampleName); + String collectionName = "schemaless".equals(exampleName) ? "gettingstarted" : exampleName; + String configSet = + "techproducts".equals(exampleName) ? "sample_techproducts_configs" : "data_driven_schema_configs"; + + boolean isCloudMode = cli.hasOption('c'); + String zkHost = cli.getOptionValue('z'); + int port = Integer.parseInt(cli.getOptionValue('p', "8983")); + Map nodeStatus = startSolr(new File(exDir, "solr"), isCloudMode, cli, port, zkHost, 30); + + // invoke the CreateTool + File configsetsDir = new File(serverDir, "solr/configsets"); + + String solrUrl = (String)nodeStatus.get("baseUrl"); + + // safe check if core / collection already exists + boolean alreadyExists = false; + if (nodeStatus.get("cloud") != null) { + String collectionListUrl = solrUrl+"/admin/collections?action=list"; + if (safeCheckCollectionExists(collectionListUrl, collectionName)) { + alreadyExists = true; + echo("\nWARNING: Collection '"+collectionName+ + "' already exists!\nChecked collection existence using Collections API command:\n"+collectionListUrl+"\n"); + } + } else { + String coreName = collectionName; + String coreStatusUrl = solrUrl+"/admin/cores?action=STATUS&core="+coreName; + if (safeCheckCoreExists(coreStatusUrl, coreName)) { + alreadyExists = true; + echo("\nWARNING: Core '" + coreName + + "' already exists!\nChecked core existence using Core API command:\n" + coreStatusUrl+"\n"); + } + } + + if (!alreadyExists) { + String[] createArgs = new String[] { + "-name", collectionName, + "-shards", "1", + "-replicationFactor", "1", + "-confname", collectionName, + "-confdir", configSet, + "-configsetsDir", configsetsDir.getAbsolutePath(), + "-solrUrl", solrUrl + }; + CreateTool createTool = new CreateTool(stdout); + int createCode = + createTool.runTool(processCommandLineArgs(joinCommonAndToolOptions(createTool.getOptions()), createArgs)); + if (createCode != 0) + throw new Exception("Failed to create "+collectionName+" using command: "+ Arrays.asList(createArgs)); + } + + if ("techproducts".equals(exampleName)) { + + File exampledocsDir = new File(exampleDir, "exampledocs"); + if (!exampledocsDir.isDirectory()) { + File readOnlyExampleDir = new File(serverDir.getParentFile(), "example"); + if (readOnlyExampleDir.isDirectory()) { + exampledocsDir = new File(readOnlyExampleDir, "exampledocs"); + } + } + + if (exampledocsDir.isDirectory()) { + File postJarFile = new File(exampledocsDir, "post.jar"); + if (postJarFile.isFile()) { + echo("Indexing tech product example docs from "+exampledocsDir.getAbsolutePath()); + + String javaHome = System.getProperty("java.home"); + String java = javaHome+"/bin/java"; + String updateUrl = String.format(Locale.ROOT, "%s/%s/update", solrUrl, collectionName); + + String postCmd = String.format(Locale.ROOT, "%s -Durl=\"%s\" -jar %s \"%s\"/*.xml", + java, updateUrl, postJarFile.getAbsolutePath(), exampledocsDir.getAbsolutePath()); + + executor.execute(org.apache.commons.exec.CommandLine.parse(postCmd)); + } else { + echo("example/exampledocs/post.jar not found, skipping indexing step for the techproducts example"); + } + } else { + echo("exampledocs directory not found, skipping indexing step for the techproducts example"); + } + } + + echo("\nSolr "+exampleName+" example launched successfully. Direct your Web browser to "+solrUrl+" to visit the Solr Admin UI"); + } + + protected void runCloudExample(CommandLine cli) throws Exception { + + boolean prompt = !cli.hasOption("noprompt"); + int numNodes = 2; + int[] cloudPorts = new int[]{ 8983, 7574, 8984, 7575 }; + File cloudDir = new File(exampleDir, "cloud"); + if (!cloudDir.isDirectory()) + cloudDir.mkdir(); + + echo("\nWelcome to the SolrCloud example!\n"); + + Scanner readInput = prompt ? new Scanner(userInput, StandardCharsets.UTF_8.name()) : null; + if (prompt) { + echo("This interactive session will help you launch a SolrCloud cluster on your local workstation."); + + // get the number of nodes to start + numNodes = promptForInt(readInput, + "To begin, how many Solr nodes would you like to run in your local cluster? (specify 1-4 nodes) [2]: ", + "a number", numNodes, 1, 4); + + echo("Ok, let's start up "+numNodes+" Solr nodes for your example SolrCloud cluster."); + + // get the ports for each port + for (int n=0; n < numNodes; n++) { + String promptMsg = + String.format(Locale.ROOT, "Please enter the port for node%d [%d]: ", (n+1), cloudPorts[n]); + int port = promptForPort(readInput, n+1, promptMsg, cloudPorts[n]); + while (!isPortAvailable(port)) { + port = promptForPort(readInput, n+1, + "Oops! Looks like port "+port+ + " is already being used by another process. Please choose a different port.", cloudPorts[n]); + } + + cloudPorts[n] = port; + if (verbose) + echo("Using port "+port+" for node "+(n+1)); + } + } else { + echo("Starting up "+numNodes+" Solr nodes for your example SolrCloud cluster.\n"); + } + + // setup a unique solr.solr.home directory for each node + File node1Dir = setupExampleDir(serverDir, cloudDir, "node1"); + for (int n=2; n <= numNodes; n++) { + File nodeNDir = new File(cloudDir, "node"+n); + if (!nodeNDir.isDirectory()) { + echo("Cloning " + node1Dir.getAbsolutePath() + " into\n "+nodeNDir.getAbsolutePath()); + FileUtils.copyDirectory(node1Dir, nodeNDir); + } else { + echo(nodeNDir.getAbsolutePath()+" already exists."); + } + } + + // deal with extra args passed to the script to run the example + String zkHost = cli.getOptionValue('z'); + + // start the first node (most likely with embedded ZK) + Map nodeStatus = + startSolr(new File(node1Dir,"solr"), true, cli, cloudPorts[0], zkHost, 30); + + if (zkHost == null) { + Map cloudStatus = (Map)nodeStatus.get("cloud"); + if (cloudStatus != null) { + String zookeeper = (String)cloudStatus.get("ZooKeeper"); + if (zookeeper != null) + zkHost = zookeeper; + } + if (zkHost == null) + throw new Exception("Could not get the ZooKeeper connection string for node1!"); + } + + if (numNodes > 1) { + // start the other nodes + for (int n = 1; n < numNodes; n++) + startSolr(new File(cloudDir, "node"+(n+1)+"/solr"), true, cli, cloudPorts[n], zkHost, 30); + } + + String solrUrl = (String)nodeStatus.get("baseUrl"); + if (solrUrl.endsWith("/")) + solrUrl = solrUrl.substring(0,solrUrl.length()-1); + + // wait until live nodes == numNodes + waitToSeeLiveNodes(10 /* max wait */, zkHost, numNodes); + + // create the collection + String collectionName = + createCloudExampleCollection(numNodes, readInput, prompt, solrUrl); + + // update the config to enable soft auto-commit + echo("\nEnabling auto soft-commits with maxTime 3 secs using the Config API"); + setCollectionConfigProperty(solrUrl, collectionName, "updateHandler.autoSoftCommit.maxTime", "3000"); + + echo("\n\nSolrCloud example running, please visit: "+solrUrl+" \n"); + } + + protected void setCollectionConfigProperty(String solrUrl, String collectionName, String propName, String propValue) { + ConfigTool configTool = new ConfigTool(stdout); + String[] configArgs = + new String[] { "-collection", collectionName, "-property", propName, "-value", propValue, "-solrUrl", solrUrl }; + + // let's not fail if we get this far ... just report error and finish up + try { + configTool.runTool(processCommandLineArgs(joinCommonAndToolOptions(configTool.getOptions()), configArgs)); + } catch (Exception exc) { + System.err.println("Failed to update '"+propName+"' property due to: "+exc); + } + } + + protected void waitToSeeLiveNodes(int maxWaitSecs, String zkHost, int numNodes) { + CloudSolrClient cloudClient = null; + try { + cloudClient = new CloudSolrClient(zkHost); + cloudClient.connect(); + Set liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes(); + int numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0; + long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(maxWaitSecs, TimeUnit.SECONDS); + while (System.nanoTime() < timeout && numLiveNodes < numNodes) { + echo("\nWaiting up to "+maxWaitSecs+" seconds to see "+ + (numNodes-numLiveNodes)+" more nodes join the SolrCloud cluster ..."); + try { + Thread.sleep(2000); + } catch (InterruptedException ie) { + Thread.interrupted(); + } + liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes(); + numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0; + } + if (numLiveNodes < numNodes) { + echo("\nWARNING: Only "+numLiveNodes+" of "+numNodes+ + " are active in the cluster after "+maxWaitSecs+ + " seconds! Please check the solr.log for each node to look for errors.\n"); + } + } catch (Exception exc) { + System.err.println("Failed to see if "+numNodes+" joined the SolrCloud cluster due to: "+exc); + } finally { + if (cloudClient != null) { + try { + cloudClient.close(); + } catch (Exception ignore) {} + } + } + } + + protected Map startSolr(File solrHomeDir, + boolean cloudMode, + CommandLine cli, + int port, + String zkHost, + int maxWaitSecs) + throws Exception + { + + String extraArgs = readExtraArgs(cli.getArgs()); + + String host = cli.getOptionValue('h'); + String memory = cli.getOptionValue('m'); + + String hostArg = (host != null && !"localhost".equals(host)) ? " -h "+host : ""; + String zkHostArg = (zkHost != null) ? " -z "+zkHost : ""; + String memArg = (memory != null) ? " -m "+memory : ""; + String cloudModeArg = cloudMode ? "-cloud " : ""; + + String addlOpts = cli.getOptionValue('a'); + String addlOptsArg = (addlOpts != null) ? " -a \""+addlOpts+"\"" : ""; + + File cwd = new File(System.getProperty("user.dir")); + File binDir = (new File(script)).getParentFile(); + + boolean isWindows = (OS.isFamilyDOS() || OS.isFamilyWin9x() || OS.isFamilyWindows()); + String callScript = (!isWindows && cwd.equals(binDir.getParentFile())) ? "bin/solr" : script; + + String cwdPath = cwd.getAbsolutePath(); + String solrHome = solrHomeDir.getAbsolutePath(); + + // don't display a huge path for solr home if it is relative to the cwd + if (!isWindows && solrHome.startsWith(cwdPath)) + solrHome = solrHome.substring(cwdPath.length()+1); + + String startCmd = + String.format(Locale.ROOT, "%s start %s -p %d -s \"%s\" %s %s %s %s %s", + callScript, cloudModeArg, port, solrHome, hostArg, zkHostArg, memArg, extraArgs, addlOptsArg); + startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing + + echo("\nStarting up Solr on port " + port + " using command:"); + echo(startCmd + "\n"); + + String solrUrl = + String.format(Locale.ROOT, "%s://%s:%d/solr", urlScheme, (host != null ? host : "localhost"), port); + + Map nodeStatus = checkPortConflict(solrUrl, solrHomeDir, port, cli); + if (nodeStatus != null) + return nodeStatus; // the server they are trying to start is already running + + int code = 0; + if (isWindows) { + // On Windows, the execution doesn't return, so we have to execute async + // and when calling the script, it seems to be inheriting the environment that launched this app + // so we have to prune out env vars that may cause issues + Map startEnv = new HashMap<>(); + Map procEnv = EnvironmentUtils.getProcEnvironment(); + if (procEnv != null) { + for (String envVar : procEnv.keySet()) { + String envVarVal = procEnv.get(envVar); + if (envVarVal != null && !"EXAMPLE".equals(envVar) && !envVar.startsWith("SOLR_")) { + startEnv.put(envVar, envVarVal); + } + } + } + executor.execute(org.apache.commons.exec.CommandLine.parse(startCmd), startEnv, new DefaultExecuteResultHandler()); + + // brief wait before proceeding on Windows + try { + Thread.sleep(3000); + } catch (InterruptedException ie) { + // safe to ignore ... + Thread.interrupted(); + } + + } else { + code = executor.execute(org.apache.commons.exec.CommandLine.parse(startCmd)); + } + if (code != 0) + throw new Exception("Failed to start Solr using command: "+startCmd); + + return getNodeStatus(solrUrl, maxWaitSecs); + } + + protected Map checkPortConflict(String solrUrl, File solrHomeDir, int port, CommandLine cli) { + // quickly check if the port is in use + if (isPortAvailable(port)) + return null; // not in use ... try to start + + Map nodeStatus = null; + try { + nodeStatus = (new StatusTool()).getStatus(solrUrl); + } catch (Exception ignore) { /* just trying to determine if this example is already running. */ } + + if (nodeStatus != null) { + String solr_home = (String)nodeStatus.get("solr_home"); + if (solr_home != null) { + String solrHomePath = solrHomeDir.getAbsolutePath(); + if (!solrHomePath.endsWith("/")) + solrHomePath += "/"; + if (!solr_home.endsWith("/")) + solr_home += "/"; + + if (solrHomePath.equals(solr_home)) { + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(nodeStatus); + echo("Solr is already setup and running on port " + port + " with status:\n" + arr.toString()); + echo("\nIf this is not the example node you are trying to start, please choose a different port."); + nodeStatus.put("baseUrl", solrUrl); + return nodeStatus; + } + } + } + + throw new IllegalStateException("Port "+port+" is already being used by another process."); + } + + protected String readExtraArgs(String[] extraArgsArr) { + String extraArgs = ""; + if (extraArgsArr != null && extraArgsArr.length > 0) { + StringBuilder sb = new StringBuilder(); + int app = 0; + for (int e=0; e < extraArgsArr.length; e++) { + String arg = extraArgsArr[e]; + if ("e".equals(arg) || "example".equals(arg)) { + e++; // skip over the example arg + continue; + } + + if (app > 0) sb.append(" "); + sb.append(arg); + ++app; + } + extraArgs = sb.toString().trim(); + } + return extraArgs; + } + + protected String createCloudExampleCollection(int numNodes, Scanner readInput, boolean prompt, String solrUrl) throws Exception { + // yay! numNodes SolrCloud nodes running + int numShards = 2; + int replicationFactor = 2; + String cloudConfig = "data_driven_schema_configs"; + String collectionName = "gettingstarted"; + + File configsetsDir = new File(serverDir, "solr/configsets"); + String collectionListUrl = solrUrl+"/admin/collections?action=list"; + + if (prompt) { + echo("\nNow let's create a new collection for indexing documents in your "+numNodes+"-node cluster."); + + while (true) { + collectionName = + prompt(readInput, "Please provide a name for your new collection: ["+collectionName+"] ", collectionName); + + // Test for existence and then prompt to either create another or skip the create step + if (safeCheckCollectionExists(collectionListUrl, collectionName)) { + echo("\nCollection '"+collectionName+"' already exists!"); + int oneOrTwo = promptForInt(readInput, + "Do you want to re-use the existing collection or create a new one? Enter 1 to reuse, 2 to create new [1]: ", "a 1 or 2", 1, 1, 2); + if (oneOrTwo == 1) { + return collectionName; + } else { + continue; + } + } else { + break; // user selected a collection that doesn't exist ... proceed on + } + } + + numShards = promptForInt(readInput, + "How many shards would you like to split " + collectionName + " into? [2]", "a shard count", 2, 1, 4); + + replicationFactor = promptForInt(readInput, + "How many replicas per shard would you like to create? [2] ", "a replication factor", 2, 1, 4); + + echo("Please choose a configuration for the "+collectionName+" collection, available options are:"); + cloudConfig = + prompt(readInput, "basic_configs, data_driven_schema_configs, or sample_techproducts_configs ["+cloudConfig+"] ", cloudConfig); + + // validate the cloudConfig name + while (!isValidConfig(configsetsDir, cloudConfig)) { + echo(cloudConfig+" is not a valid configuration directory! Please choose a configuration for the "+collectionName+" collection, available options are:"); + cloudConfig = + prompt(readInput, "basic_configs, data_driven_schema_configs, or sample_techproducts_configs ["+cloudConfig+"] ", cloudConfig); + } + } else { + // must verify if default collection exists + if (safeCheckCollectionExists(collectionListUrl, collectionName)) { + echo("\nCollection '"+collectionName+"' already exists! Skipping collection creation step."); + return collectionName; + } + } + + // invoke the CreateCollectionTool + String[] createArgs = new String[] { + "-name", collectionName, + "-shards", String.valueOf(numShards), + "-replicationFactor", String.valueOf(replicationFactor), + "-confname", collectionName, + "-confdir", cloudConfig, + "-configsetsDir", configsetsDir.getAbsolutePath(), + "-solrUrl", solrUrl + }; + + CreateCollectionTool createCollectionTool = new CreateCollectionTool(stdout); + int createCode = + createCollectionTool.runTool( + processCommandLineArgs(joinCommonAndToolOptions(createCollectionTool.getOptions()), createArgs)); + + if (createCode != 0) + throw new Exception("Failed to create collection using command: "+ Arrays.asList(createArgs)); + + return collectionName; + } + + protected boolean isValidConfig(File configsetsDir, String config) { + File configDir = new File(configsetsDir, config); + if (configDir.isDirectory()) + return true; + + // not a built-in configset ... maybe it's a custom directory? + configDir = new File(config); + if (configDir.isDirectory()) + return true; + + return false; + } + + protected Map getNodeStatus(String solrUrl, int maxWaitSecs) throws Exception { + StatusTool statusTool = new StatusTool(); + if (verbose) + echo("\nChecking status of Solr at " + solrUrl + " ..."); + + URL solrURL = new URL(solrUrl); + Map nodeStatus = statusTool.waitToSeeSolrUp(solrUrl, maxWaitSecs); + nodeStatus.put("baseUrl", solrUrl); + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(nodeStatus); + String mode = (nodeStatus.get("cloud") != null) ? "cloud" : "standalone"; + if (verbose) + echo("\nSolr is running on "+solrURL.getPort()+" in " + mode + " mode with status:\n" + arr.toString()); + + return nodeStatus; + } + + protected File setupExampleDir(File serverDir, File exampleParentDir, String dirName) throws IOException { + File solrXml = new File(serverDir, "solr/solr.xml"); + if (!solrXml.isFile()) + throw new IllegalArgumentException("Value of -serverDir option is invalid! "+ + solrXml.getAbsolutePath()+" not found!"); + + File zooCfg = new File(serverDir, "solr/zoo.cfg"); + if (!zooCfg.isFile()) + throw new IllegalArgumentException("Value of -serverDir option is invalid! "+ + zooCfg.getAbsolutePath()+" not found!"); + + File solrHomeDir = new File(exampleParentDir, dirName+"/solr"); + if (!solrHomeDir.isDirectory()) { + echo("Creating Solr home directory "+solrHomeDir); + solrHomeDir.mkdirs(); + } else { + echo("Solr home directory "+solrHomeDir.getAbsolutePath()+" already exists."); + } + + copyIfNeeded(solrXml, new File(solrHomeDir, "solr.xml")); + copyIfNeeded(zooCfg, new File(solrHomeDir, "zoo.cfg")); + + return solrHomeDir.getParentFile(); + } + + protected void copyIfNeeded(File src, File dest) throws IOException { + if (!dest.isFile()) + FileUtils.copyFile(src, dest); + + if (!dest.isFile()) + throw new IllegalStateException("Required file "+dest.getAbsolutePath()+" not found!"); + } + + protected boolean isPortAvailable(int port) { + Socket s = null; + try { + s = new Socket("localhost", port); + return false; + } catch (IOException e) { + return true; + } finally { + if (s != null) { + try { + s.close(); + } catch (IOException ignore) {} + } + } + } + + protected Integer promptForPort(Scanner s, int node, String prompt, Integer defVal) { + return promptForInt(s, prompt, "a port for node "+node, defVal, null, null); + } + + protected Integer promptForInt(Scanner s, String prompt, String label, Integer defVal, Integer min, Integer max) { + Integer inputAsInt = null; + + String value = prompt(s, prompt, null /* default is null since we handle that here */); + if (value != null) { + int attempts = 3; + while (value != null && --attempts > 0) { + try { + inputAsInt = new Integer(value); + + if (min != null) { + if (inputAsInt < min) { + value = prompt(s, String.format(Locale.ROOT, PROMPT_NUMBER_TOO_SMALL, inputAsInt, label, min, max, defVal)); + inputAsInt = null; + continue; + } + } + + if (max != null) { + if (inputAsInt > max) { + value = prompt(s, String.format(Locale.ROOT, PROMPT_NUMBER_TOO_LARGE, inputAsInt, label, min, max, defVal)); + inputAsInt = null; + continue; + } + } + + } catch (NumberFormatException nfe) { + if (verbose) + echo(value+" is not a number!"); + + if (min != null && max != null) { + value = prompt(s, String.format(Locale.ROOT, PROMPT_FOR_NUMBER_IN_RANGE, label, min, max, defVal)); + } else { + value = prompt(s, String.format(Locale.ROOT, PROMPT_FOR_NUMBER, label, defVal)); + } + } + } + if (attempts == 0 && value != null && inputAsInt == null) + echo("Too many failed attempts! Going with default value "+defVal); + } + + return (inputAsInt != null) ? inputAsInt : defVal; + } + + protected String prompt(Scanner s, String prompt) { + return prompt(s, prompt, null); + } + + protected String prompt(Scanner s, String prompt, String defaultValue) { + echo(prompt); + String nextInput = s.nextLine(); + if (nextInput != null) { + nextInput = nextInput.trim(); + if (nextInput.isEmpty()) + nextInput = null; + } + return (nextInput != null) ? nextInput : defaultValue; + } + + } // end RunExampleTool class } diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java index 7284cc730a4..0d261dcd45a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java @@ -69,7 +69,6 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase { // create the gettingstarted collection just like the bin/solr script would do String[] args = new String[] { - "create_collection", "-name", testCollectionName, "-shards", "2", "-replicationFactor", "2", @@ -78,6 +77,9 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase { "-configsetsDir", data_driven_schema_configs.getParentFile().getParentFile().getAbsolutePath(), "-solrUrl", solrUrl }; + + // NOTE: not calling SolrCLI.main as the script does because it calls System.exit which is a no-no in a JUnit test + SolrCLI.CreateCollectionTool tool = new SolrCLI.CreateCollectionTool(); CommandLine cli = SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args); log.info("Creating the '"+testCollectionName+"' collection using SolrCLI with: "+solrUrl); @@ -147,7 +149,6 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase { protected void doTestHealthcheck(String testCollectionName, String zkHost) throws Exception { String[] args = new String[]{ - "healthcheck", "-collection", testCollectionName, "-zkHost", zkHost }; @@ -159,7 +160,6 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase { protected void doTestDeleteAction(String testCollectionName, String solrUrl) throws Exception { String[] args = new String[] { - "delete", "-name", testCollectionName, "-solrUrl", solrUrl }; @@ -186,7 +186,6 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase { String prop = "updateHandler.autoSoftCommit.maxTime"; Long maxTime = new Long(3000L); String[] args = new String[]{ - "config", "-collection", testCollectionName, "-property", prop, "-value", maxTime.toString(), diff --git a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java new file mode 100644 index 00000000000..271c24d5202 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java @@ -0,0 +1,485 @@ +package org.apache.solr.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteResultHandler; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.embedded.JettyConfig; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.common.SolrInputDocument; +import org.junit.After; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests the SolrCLI.RunExampleTool implementation that supports bin/solr -e [example] + */ +@LuceneTestCase.Slow +@SolrTestCaseJ4.SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") +public class TestSolrCLIRunExample extends SolrTestCaseJ4 { + + protected static final transient Logger log = LoggerFactory.getLogger(TestSolrCLIRunExample.class); + + /** + * Overrides the call to exec bin/solr to start Solr nodes to start them using the Solr test-framework + * instead of the script, since the script depends on a full build. + */ + private class RunExampleExecutor extends DefaultExecutor implements Closeable { + + private PrintStream stdout; + private List commandsExecuted = new ArrayList<>(); + private MiniSolrCloudCluster solrCloudCluster; + private JettySolrRunner standaloneSolr; + + RunExampleExecutor(PrintStream stdout) { + super(); + this.stdout = stdout; + } + + /** + * Override the call to execute a command asynchronously to occur synchronously during a unit test. + */ + @Override + public void execute(org.apache.commons.exec.CommandLine cmd, Map env, ExecuteResultHandler erh) throws IOException { + int code = execute(cmd); + if (code != 0) throw new RuntimeException("Failed to execute cmd: "+joinArgs(cmd.getArguments())); + } + + @Override + public int execute(org.apache.commons.exec.CommandLine cmd) throws IOException { + // collect the commands as they are executed for analysis by the test + commandsExecuted.add(cmd); + + String exe = cmd.getExecutable(); + if (exe.endsWith("solr")) { + String[] args = cmd.getArguments(); + if ("start".equals(args[0])) { + if (!hasFlag("-cloud", args) && !hasFlag("-c", args)) + return startStandaloneSolr(args); + + File baseDir = createTempDir().toFile(); + File solrHomeDir = new File(getArg("-s", args)); + int port = Integer.parseInt(getArg("-p", args)); + + JettyConfig jettyConfig = + JettyConfig.builder().setContext("/solr").setPort(port).build(); + try { + if (solrCloudCluster == null) { + System.setProperty("host", "localhost"); + System.setProperty("jetty.port", String.valueOf(port)); + solrCloudCluster = + new MiniSolrCloudCluster(1, baseDir, new File(solrHomeDir, "solr.xml"), jettyConfig); + } else { + // another member of this cluster -- not supported yet, due to how MiniSolrCloudCluster works + throw new IllegalArgumentException("Only launching one SolrCloud node is supported by this test!"); + } + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException)e; + } else { + throw new RuntimeException(e); + } + } + } else if ("stop".equals(args[0])) { + + int port = Integer.parseInt(getArg("-p", args)); + + // stop the requested node + if (standaloneSolr != null) { + int localPort = standaloneSolr.getLocalPort(); + if (port == localPort) { + try { + standaloneSolr.stop(); + log.info("Stopped standalone Solr instance running on port "+port); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException)e; + } else { + throw new RuntimeException(e); + } + } + } else { + throw new IllegalArgumentException("No Solr is running on port "+port); + } + } else { + if (solrCloudCluster != null) { + try { + solrCloudCluster.shutdown(); + log.info("Stopped SolrCloud test cluster"); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException)e; + } else { + throw new RuntimeException(e); + } + } + } else { + throw new IllegalArgumentException("No Solr nodes found to stop!"); + } + } + } + } else { + String cmdLine = joinArgs(cmd.getArguments()); + if (cmdLine.indexOf("post.jar") != -1) { + // invocation of the post.jar file ... we'll just hit the SimplePostTool directly vs. trying to invoke another JVM + List argsToSimplePostTool = new ArrayList(); + boolean afterPostJarArg = false; + for (String arg : cmd.getArguments()) { + if (arg.startsWith("-D")) { + arg = arg.substring(2); + int eqPos = arg.indexOf("="); + System.setProperty(arg.substring(0,eqPos), arg.substring(eqPos+1)); + } else { + if (arg.endsWith("post.jar")) { + afterPostJarArg = true; + } else { + if (afterPostJarArg) { + argsToSimplePostTool.add(arg); + } + } + } + } + SimplePostTool.main(argsToSimplePostTool.toArray(new String[0])); + } else { + log.info("Executing command: "+cmdLine); + try { + return super.execute(cmd); + } catch (Exception exc) { + log.error("Execute command ["+cmdLine+"] failed due to: "+exc, exc); + throw exc; + } + } + } + + return 0; + } + + protected String joinArgs(String[] args) { + if (args == null || args.length == 0) + return ""; + + StringBuilder sb = new StringBuilder(); + for (int a=0; a < args.length; a++) { + if (a > 0) sb.append(' '); + sb.append(args[a]); + } + return sb.toString(); + } + + protected int startStandaloneSolr(String[] args) { + + if (standaloneSolr != null) { + throw new IllegalStateException("Test is already running a standalone Solr instance "+ + standaloneSolr.getBaseUrl()+"! This indicates a bug in the unit test logic."); + } + + if (solrCloudCluster != null) { + throw new IllegalStateException("Test is already running a mini SolrCloud cluster! "+ + "This indicates a bug in the unit test logic."); + } + + int port = Integer.parseInt(getArg("-p", args)); + + File solrHomeDir = new File(getArg("-s", args)); + + System.setProperty("host", "localhost"); + System.setProperty("jetty.port", String.valueOf(port)); + + standaloneSolr = new JettySolrRunner(solrHomeDir.getAbsolutePath(), "/solr", port); + Thread bg = new Thread() { + public void run() { + try { + standaloneSolr.start(); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException)e; + } else { + throw new RuntimeException(e); + } + } + } + }; + bg.start(); + + return 0; + } + + protected String getArg(String arg, String[] args) { + for (int a=0; a < args.length; a++) { + if (arg.equals(args[a])) { + if (a+1 >= args.length) + throw new IllegalArgumentException("Missing required value for the "+arg+" option!"); + + return args[a + 1]; + } + } + throw new IllegalArgumentException("Missing required arg "+arg+ + " needed to execute command: "+commandsExecuted.get(commandsExecuted.size()-1)); + } + + protected boolean hasFlag(String flag, String[] args) { + for (String arg : args) { + if (flag.equals(arg)) + return true; + } + return false; + } + + @Override + public void close() throws IOException { + if (solrCloudCluster != null) { + try { + solrCloudCluster.shutdown(); + } catch (Exception e) { + log.warn("Failed to shutdown MiniSolrCloudCluster due to: " + e); + } + } + + if (standaloneSolr != null) { + try { + standaloneSolr.stop(); + } catch (Exception exc) { + log.warn("Failed to shutdown standalone Solr due to: " + exc); + } + standaloneSolr = null; + } + } + } + + protected List closeables = new ArrayList<>(); + + @After + public void tearDown() throws Exception { + super.tearDown(); + + if (closeables != null) { + for (Closeable toClose : closeables) { + try { + toClose.close(); + } catch (Exception ignore) {} + } + closeables.clear(); + closeables = null; + } + } + + @Test + public void testTechproductsExample() throws Exception { + testExample("techproducts"); + } + + @Test + public void testSchemalessExample() throws Exception { + testExample("schemaless"); + } + + protected void testExample(String exampleName) throws Exception { + File solrHomeDir = new File(ExternalPaths.SERVER_HOME); + if (!solrHomeDir.isDirectory()) + fail(solrHomeDir.getAbsolutePath()+" not found and is required to run this test!"); + + Path tmpDir = createTempDir(); + File solrExampleDir = tmpDir.toFile(); + File solrServerDir = solrHomeDir.getParentFile(); + + // need a port to start the example server on + int bindPort = -1; + try (ServerSocket socket = new ServerSocket(0)) { + bindPort = socket.getLocalPort(); + } + + log.info("Selected port "+bindPort+" to start "+exampleName+" example Solr instance on ..."); + + String[] toolArgs = new String[] { + "-e", exampleName, + "-serverDir", solrServerDir.getAbsolutePath(), + "-exampleDir", solrExampleDir.getAbsolutePath(), + "-p", String.valueOf(bindPort) + }; + + // capture tool output to stdout + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream stdoutSim = new PrintStream(baos, true, StandardCharsets.UTF_8.name()); + + RunExampleExecutor executor = new RunExampleExecutor(stdoutSim); + closeables.add(executor); + + SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, System.in, stdoutSim); + try { + tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs)); + } catch (Exception e) { + log.error("RunExampleTool failed due to: " + e + + "; stdout from tool prior to failure: " + baos.toString(StandardCharsets.UTF_8.name())); + throw e; + } + + String toolOutput = baos.toString(StandardCharsets.UTF_8.name()); + + // dump all the output written by the SolrCLI commands to stdout + //System.out.println("\n\n"+toolOutput+"\n\n"); + + File exampleSolrHomeDir = new File(solrExampleDir, exampleName+"/solr"); + assertTrue(exampleSolrHomeDir.getAbsolutePath() + " not found! run " + + exampleName + " example failed; output: " + toolOutput, + exampleSolrHomeDir.isDirectory()); + + if ("techproducts".equals(exampleName)) { + HttpSolrClient solrClient = new HttpSolrClient("http://localhost:" + bindPort + "/solr/" + exampleName); + SolrQuery query = new SolrQuery("*:*"); + QueryResponse qr = solrClient.query(query); + long numFound = qr.getResults().getNumFound(); + assertTrue("expected 32 docs in the "+exampleName+" example but found " + numFound + ", output: " + toolOutput, + numFound == 32); + } + + // stop the test instance + executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p " + bindPort)); + } + + /** + * Tests the interactive SolrCloud example; we cannot test the non-interactive because we need control over + * the port and can only test with one node since the test relies on setting the host and jetty.port system + * properties, i.e. there is no test coverage for the -noprompt option. + */ + @Test + public void testInteractiveSolrCloudExample() throws Exception { + File solrHomeDir = new File(ExternalPaths.SERVER_HOME); + if (!solrHomeDir.isDirectory()) + fail(solrHomeDir.getAbsolutePath()+" not found and is required to run this test!"); + + Path tmpDir = createTempDir(); + File solrExampleDir = tmpDir.toFile(); + + File solrServerDir = solrHomeDir.getParentFile(); + + String[] toolArgs = new String[] { + "-example", "cloud", + "-serverDir", solrServerDir.getAbsolutePath(), + "-exampleDir", solrExampleDir.getAbsolutePath() + }; + + int bindPort = -1; + try (ServerSocket socket = new ServerSocket(0)) { + bindPort = socket.getLocalPort(); + } + + String collectionName = "testCloudExamplePrompt"; + + // sthis test only support launching one SolrCloud node due to how MiniSolrCloudCluster works + // and the need for setting the host and port system properties ... + String userInput = "1\n"+bindPort+"\n"+collectionName+"\n2\n2\ndata_driven_schema_configs\n"; + + // simulate user input from stdin + InputStream userInputSim = new ByteArrayInputStream(userInput.getBytes(StandardCharsets.UTF_8)); + + // capture tool output to stdout + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream stdoutSim = new PrintStream(baos, true, StandardCharsets.UTF_8.name()); + + RunExampleExecutor executor = new RunExampleExecutor(stdoutSim); + closeables.add(executor); + + SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, userInputSim, stdoutSim); + try { + tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs)); + } catch (Exception e) { + System.err.println("RunExampleTool failed due to: " + e + + "; stdout from tool prior to failure: " + baos.toString(StandardCharsets.UTF_8.name())); + throw e; + } + + String toolOutput = baos.toString(StandardCharsets.UTF_8.name()); + + // verify Solr is running on the expected port and verify the collection exists + String solrUrl = "http://localhost:"+bindPort+"/solr"; + String collectionListUrl = solrUrl+"/admin/collections?action=list"; + if (!SolrCLI.safeCheckCollectionExists(collectionListUrl, collectionName)) { + fail("After running Solr cloud example, test collection '"+collectionName+ + "' not found in Solr at: "+solrUrl+"; tool output: "+toolOutput); + } + + // index some docs - to verify all is good for both shards + CloudSolrClient cloudClient = null; + + try { + cloudClient = new CloudSolrClient(executor.solrCloudCluster.getZkServer().getZkAddress()); + cloudClient.connect(); + cloudClient.setDefaultCollection(collectionName); + + int numDocs = 10; + for (int d=0; d < numDocs; d++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.setField("id", "doc"+d); + doc.setField("str_s", "a"); + cloudClient.add(doc); + } + cloudClient.commit(); + + QueryResponse qr = cloudClient.query(new SolrQuery("str_s:a")); + if (qr.getResults().getNumFound() != numDocs) { + fail("Expected "+numDocs+" to be found in the "+collectionName+ + " collection but only found "+qr.getResults().getNumFound()); + } + } finally { + if (cloudClient != null) { + try { + cloudClient.close(); + } catch (Exception ignore){} + } + } + + File node1SolrHome = new File(solrExampleDir, "cloud/node1/solr"); + if (!node1SolrHome.isDirectory()) { + fail(node1SolrHome.getAbsolutePath()+" not found! run cloud example failed; tool output: "+toolOutput); + } + + // delete the collection + SolrCLI.DeleteTool deleteTool = new SolrCLI.DeleteTool(stdoutSim); + String[] deleteArgs = new String[] { "-name", collectionName, "-solrUrl", solrUrl }; + deleteTool.runTool( + SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs)); + + // dump all the output written by the SolrCLI commands to stdout + //System.out.println(toolOutput); + + // stop the test instance + executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p "+bindPort)); + } +} diff --git a/solr/licenses/commons-exec-1.3.jar.sha1 b/solr/licenses/commons-exec-1.3.jar.sha1 new file mode 100644 index 00000000000..fca1c0110ff --- /dev/null +++ b/solr/licenses/commons-exec-1.3.jar.sha1 @@ -0,0 +1 @@ +8dfb9facd0830a27b1b5f29f84593f0aeee7773b diff --git a/solr/licenses/commons-exec-LICENSE-ASL.txt b/solr/licenses/commons-exec-LICENSE-ASL.txt new file mode 100644 index 00000000000..f820d4bd3a6 --- /dev/null +++ b/solr/licenses/commons-exec-LICENSE-ASL.txt @@ -0,0 +1,203 @@ +/* + * Apache License + * Version 2.0, January 2004 + * http://www.apache.org/licenses/ + * + * TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + * + * 1. Definitions. + * + * "License" shall mean the terms and conditions for use, reproduction, + * and distribution as defined by Sections 1 through 9 of this document. + * + * "Licensor" shall mean the copyright owner or entity authorized by + * the copyright owner that is granting the License. + * + * "Legal Entity" shall mean the union of the acting entity and all + * other entities that control, are controlled by, or are under common + * control with that entity. For the purposes of this definition, + * "control" means (i) the power, direct or indirect, to cause the + * direction or management of such entity, whether by contract or + * otherwise, or (ii) ownership of fifty percent (50%) or more of the + * outstanding shares, or (iii) beneficial ownership of such entity. + * + * "You" (or "Your") shall mean an individual or Legal Entity + * exercising permissions granted by this License. + * + * "Source" form shall mean the preferred form for making modifications, + * including but not limited to software source code, documentation + * source, and configuration files. + * + * "Object" form shall mean any form resulting from mechanical + * transformation or translation of a Source form, including but + * not limited to compiled object code, generated documentation, + * and conversions to other media types. + * + * "Work" shall mean the work of authorship, whether in Source or + * Object form, made available under the License, as indicated by a + * copyright notice that is included in or attached to the work + * (an example is provided in the Appendix below). + * + * "Derivative Works" shall mean any work, whether in Source or Object + * form, that is based on (or derived from) the Work and for which the + * editorial revisions, annotations, elaborations, or other modifications + * represent, as a whole, an original work of authorship. For the purposes + * of this License, Derivative Works shall not include works that remain + * separable from, or merely link (or bind by name) to the interfaces of, + * the Work and Derivative Works thereof. + * + * "Contribution" shall mean any work of authorship, including + * the original version of the Work and any modifications or additions + * to that Work or Derivative Works thereof, that is intentionally + * submitted to Licensor for inclusion in the Work by the copyright owner + * or by an individual or Legal Entity authorized to submit on behalf of + * the copyright owner. For the purposes of this definition, "submitted" + * means any form of electronic, verbal, or written communication sent + * to the Licensor or its representatives, including but not limited to + * communication on electronic mailing lists, source code control systems, + * and issue tracking systems that are managed by, or on behalf of, the + * Licensor for the purpose of discussing and improving the Work, but + * excluding communication that is conspicuously marked or otherwise + * designated in writing by the copyright owner as "Not a Contribution." + * + * "Contributor" shall mean Licensor and any individual or Legal Entity + * on behalf of whom a Contribution has been received by Licensor and + * subsequently incorporated within the Work. + * + * 2. Grant of Copyright License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * copyright license to reproduce, prepare Derivative Works of, + * publicly display, publicly perform, sublicense, and distribute the + * Work and such Derivative Works in Source or Object form. + * + * 3. Grant of Patent License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * (except as stated in this section) patent license to make, have made, + * use, offer to sell, sell, import, and otherwise transfer the Work, + * where such license applies only to those patent claims licensable + * by such Contributor that are necessarily infringed by their + * Contribution(s) alone or by combination of their Contribution(s) + * with the Work to which such Contribution(s) was submitted. If You + * institute patent litigation against any entity (including a + * cross-claim or counterclaim in a lawsuit) alleging that the Work + * or a Contribution incorporated within the Work constitutes direct + * or contributory patent infringement, then any patent licenses + * granted to You under this License for that Work shall terminate + * as of the date such litigation is filed. + * + * 4. Redistribution. You may reproduce and distribute copies of the + * Work or Derivative Works thereof in any medium, with or without + * modifications, and in Source or Object form, provided that You + * meet the following conditions: + * + * (a) You must give any other recipients of the Work or + * Derivative Works a copy of this License; and + * + * (b) You must cause any modified files to carry prominent notices + * stating that You changed the files; and + * + * (c) You must retain, in the Source form of any Derivative Works + * that You distribute, all copyright, patent, trademark, and + * attribution notices from the Source form of the Work, + * excluding those notices that do not pertain to any part of + * the Derivative Works; and + * + * (d) If the Work includes a "NOTICE" text file as part of its + * distribution, then any Derivative Works that You distribute must + * include a readable copy of the attribution notices contained + * within such NOTICE file, excluding those notices that do not + * pertain to any part of the Derivative Works, in at least one + * of the following places: within a NOTICE text file distributed + * as part of the Derivative Works; within the Source form or + * documentation, if provided along with the Derivative Works; or, + * within a display generated by the Derivative Works, if and + * wherever such third-party notices normally appear. The contents + * of the NOTICE file are for informational purposes only and + * do not modify the License. You may add Your own attribution + * notices within Derivative Works that You distribute, alongside + * or as an addendum to the NOTICE text from the Work, provided + * that such additional attribution notices cannot be construed + * as modifying the License. + * + * You may add Your own copyright statement to Your modifications and + * may provide additional or different license terms and conditions + * for use, reproduction, or distribution of Your modifications, or + * for any such Derivative Works as a whole, provided Your use, + * reproduction, and distribution of the Work otherwise complies with + * the conditions stated in this License. + * + * 5. Submission of Contributions. Unless You explicitly state otherwise, + * any Contribution intentionally submitted for inclusion in the Work + * by You to the Licensor shall be under the terms and conditions of + * this License, without any additional terms or conditions. + * Notwithstanding the above, nothing herein shall supersede or modify + * the terms of any separate license agreement you may have executed + * with Licensor regarding such Contributions. + * + * 6. Trademarks. This License does not grant permission to use the trade + * names, trademarks, service marks, or product names of the Licensor, + * except as required for reasonable and customary use in describing the + * origin of the Work and reproducing the content of the NOTICE file. + * + * 7. Disclaimer of Warranty. Unless required by applicable law or + * agreed to in writing, Licensor provides the Work (and each + * Contributor provides its Contributions) on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied, including, without limitation, any warranties or conditions + * of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + * PARTICULAR PURPOSE. You are solely responsible for determining the + * appropriateness of using or redistributing the Work and assume any + * risks associated with Your exercise of permissions under this License. + * + * 8. Limitation of Liability. In no event and under no legal theory, + * whether in tort (including negligence), contract, or otherwise, + * unless required by applicable law (such as deliberate and grossly + * negligent acts) or agreed to in writing, shall any Contributor be + * liable to You for damages, including any direct, indirect, special, + * incidental, or consequential damages of any character arising as a + * result of this License or out of the use or inability to use the + * Work (including but not limited to damages for loss of goodwill, + * work stoppage, computer failure or malfunction, or any and all + * other commercial damages or losses), even if such Contributor + * has been advised of the possibility of such damages. + * + * 9. Accepting Warranty or Additional Liability. While redistributing + * the Work or Derivative Works thereof, You may choose to offer, + * and charge a fee for, acceptance of support, warranty, indemnity, + * or other liability obligations and/or rights consistent with this + * License. However, in accepting such obligations, You may act only + * on Your own behalf and on Your sole responsibility, not on behalf + * of any other Contributor, and only if You agree to indemnify, + * defend, and hold each Contributor harmless for any liability + * incurred by, or claims asserted against, such Contributor by reason + * of your accepting any such warranty or additional liability. + * + * END OF TERMS AND CONDITIONS + * + * APPENDIX: How to apply the Apache License to your work. + * + * To apply the Apache License to your work, attach the following + * boilerplate notice, with the fields enclosed by brackets "[]" + * replaced with your own identifying information. (Don't include + * the brackets!) The text should be enclosed in the appropriate + * comment syntax for the file format. We also recommend that a + * file or class name and description of purpose be included on the + * same "printed page" as the copyright notice for easier + * identification within third-party archives. + * + * Copyright [yyyy] [name of copyright owner] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/solr/licenses/commons-exec-NOTICE.txt b/solr/licenses/commons-exec-NOTICE.txt new file mode 100644 index 00000000000..c4add835ae6 --- /dev/null +++ b/solr/licenses/commons-exec-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons Exec +Copyright 2005-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/).