From 3f566e6e91c84ea2b22d7dc8d0523a7c1c9fea1b Mon Sep 17 00:00:00 2001 From: Timothy Potter Date: Thu, 9 Oct 2014 18:42:21 +0000 Subject: [PATCH] SOLR-6549: add a -s option to set the -Dsolr.solr.home property, thus allowing multiple Solr nodes on the same host to share the same server directory -d but with different Solr home directories git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1630550 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 ++ solr/bin/solr | 107 ++++++++++++++++++++++++++++++----------- solr/bin/solr.cmd | 111 ++++++++++++++++++++++++++++++++----------- solr/bin/solr.in.cmd | 6 ++- solr/bin/solr.in.sh | 6 ++- 5 files changed, 173 insertions(+), 60 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d0d1612cc4a..5343a810e3d 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -294,6 +294,9 @@ Other Changes * SOLR-6597: SolrIndexConfig parameter in one of the SolrIndexSearcher constructor has been removed. It was just passed and never used via that constructor. (Anshum Gupta) +* SOLR-6549: bin/solr script should support a -s option to set the -Dsolr.solr.home property. + (Timothy Potter) + ================== 4.10.1 ================== Bug Fixes diff --git a/solr/bin/solr b/solr/bin/solr index 9252691bae8..d48e9d3ee89 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -79,6 +79,23 @@ else DEFAULT_SERVER_DIR=$SOLR_TIP/example fi +# If an include wasn't specified in the environment, then search for one... +if [ "x$SOLR_INCLUDE" == "x" ]; then + # Locations (in order) to use when searching for an include file. + for include in "`dirname "$0"`/solr.in.sh" \ + "$HOME/.solr.in.sh" \ + /usr/share/solr/solr.in.sh \ + /usr/local/share/solr/solr.in.sh \ + /opt/solr/solr.in.sh; do + if [ -r "$include" ]; then + . "$include" + break + fi + done +elif [ -r "$SOLR_INCLUDE" ]; then + . "$SOLR_INCLUDE" +fi + if [ "$SOLR_JAVA_HOME" != "" ]; then JAVA=$SOLR_JAVA_HOME/bin/java elif [ -n "$JAVA_HOME" ]; then @@ -108,16 +125,20 @@ function print_usage() { echo "Usage: solr COMMAND OPTIONS" echo " where COMMAND is one of: start, stop, restart, healthcheck" echo "" - echo " Example: Start Solr running in the background on port 8984:" + echo " Standalone server example (start Solr running in the background on port 8984):" echo "" echo " ./solr start -p 8984" echo "" + echo " SolrCloud example (start Solr running in SolrCloud mode using localhost:2181 to connect to ZooKeeper, with 1g max heap size and remote Java debug options enabled):" + echo "" + echo " ./solr start -c -m 1g -z localhost:2181 -a \"-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1044\"" + echo "" echo "Pass -help after any COMMAND to see command-specific usage information," - echo " such as: ./solr start -help" + echo " such as: ./solr start -help or ./solr stop -help" echo "" elif [[ "$CMD" == "start" || "$CMD" == "restart" ]]; then echo "" - echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-a \"additional-options\"] [-V]" + echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-a \"additional-options\"] [-V]" echo "" echo " -f Start Solr in foreground; default starts Solr in the background" echo " and sends stdout / stderr to solr-PORT-console.log" @@ -137,6 +158,13 @@ function print_usage() { echo " -m Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g" echo " results in: -Xms4g -Xmx4g; by default, this script sets the heap size to 512m" echo "" + echo " -s Sets the solr.solr.home system property; Solr will create core directories under" + echo " this directory. This allows you to run multiple Solr instances on the same host" + echo " while reusing the same server directory set using the -d parameter. If set, the" + echo " specified directory should contain a solr.xml file. The default value is example/solr." + echo " This parameter is ignored when running examples (-e), as the solr.solr.home depends" + echo " on which example is run." + echo "" echo " -e Name of the example to run; available examples:" echo " cloud: SolrCloud example" echo " default: Solr default example" @@ -213,7 +241,7 @@ function jetty_port() { function run_tool() { # Extract the solr.war if it hasn't been done already (so we can access the SolrCLI class) - if [ ! -d "$DEFAULT_SERVER_DIR/solr-webapp/webapp" ]; then + if [[ -e $DEFAULT_SERVER_DIR/webapps/solr.war && ! -d "$DEFAULT_SERVER_DIR/solr-webapp/webapp" ]]; then (mkdir -p $DEFAULT_SERVER_DIR/solr-webapp/webapp && cd $DEFAULT_SERVER_DIR/solr-webapp/webapp && jar xf $DEFAULT_SERVER_DIR/webapps/solr.war) fi @@ -370,23 +398,6 @@ if [ "$SCRIPT_CMD" != "stop" ] && [ "$SCRIPT_CMD" != "start" ] && [ "$SCRIPT_CMD exit 1 fi -# If an include wasn't specified in the environment, then search for one... -if [ "x$SOLR_INCLUDE" == "x" ]; then - # Locations (in order) to use when searching for an include file. - for include in "`dirname "$0"`/solr.in.sh" \ - "$HOME/.solr.in.sh" \ - /usr/share/solr/solr.in.sh \ - /usr/local/share/solr/solr.in.sh \ - /opt/solr/solr.in.sh; do - if [ -r "$include" ]; then - . "$include" - break - fi - done -elif [ -r "$SOLR_INCLUDE" ]; then - . "$SOLR_INCLUDE" -fi - # Run in foreground (default is to run in the background) FG="false" noprompt=false @@ -405,13 +416,22 @@ if [ $# -gt 0 ]; then fi # see if the arg value is relative to the tip vs full path - if [ -d "$SOLR_TIP/$2" ]; then - SOLR_SERVER_DIR="$SOLR_TIP/$2" - else + if [[ $2 != /* ]] && [[ -d "$SOLR_TIP/$2" ]]; then + SOLR_SERVER_DIR="$SOLR_TIP/$2" + else SOLR_SERVER_DIR="$2" fi shift 2 ;; + -s|-solr.home) + if [ "${2:0:1}" == "-" ]; then + print_usage "$SCRIPT_CMD" "Expected directory but found $2 instead!" + exit 1 + fi + + SOLR_HOME="$2" + shift 2 + ;; -e|-example) if [ "${2:0:1}" == "-" ]; then print_usage "$SCRIPT_CMD" "Expected example name but found $2 instead!" @@ -461,6 +481,10 @@ if [ $# -gt 0 ]; then ADDITIONAL_CMD_OPTS="$2" shift 2 ;; + -k|-key) + STOP_KEY="$2" + shift 2 + ;; -help|-usage) print_usage "$SCRIPT_CMD" exit 0 @@ -613,8 +637,22 @@ fi if [ "$SOLR_HOME" == "" ]; then SOLR_HOME="$SOLR_SERVER_DIR/solr" +else + if [[ $SOLR_HOME != /* ]] && [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then + SOLR_HOME="$SOLR_SERVER_DIR/$SOLR_HOME" + fi fi +if [ ! -e "$SOLR_HOME" ]; then + echo -e "\nSolr home directory $SOLR_HOME not found!\n" + exit 1 +fi +if [ ! -e "$SOLR_HOME/solr.xml" ]; then + echo -e "\nSolr home directory $SOLR_HOME must contain a solr.xml file!\n" + exit 1 +fi + + if [ "$STOP_KEY" == "" ]; then STOP_KEY="solrrocks" fi @@ -832,15 +870,26 @@ else SOLR_SERVER_DIR=$SOLR_TIP/node1 SOLR_HOME=$SOLR_TIP/node1/solr SOLR_PORT=${CLOUD_PORTS[0]} + if [ "$ZK_HOST" != "" ]; then DASHZ="-z $ZK_HOST" fi + if [ "$SOLR_HEAP" != "" ]; then + DASHM="-m $SOLR_HEAP" + fi + + if [ "$ADDITIONAL_CMD_OPTS" != "" ]; then + DASHA="-a $ADDITIONAL_CMD_OPTS" + fi + echo -e "\nStarting up SolrCloud node1 on port ${CLOUD_PORTS[0]} using command:\n" - echo -e "solr start -cloud -d node1 -p $SOLR_PORT $DASHZ\n\n" + echo -e "solr start -cloud -d node1 -p $SOLR_PORT $DASHZ $DASHM $DASHA\n\n" # can't launch this node in the foreground else we can't run anymore commands - launch_solr "false" "" + launch_solr "false" "$ADDITIONAL_CMD_OPTS" + + sleep 5 # if user did not define a specific -z parameter, assume embedded in first cloud node we launched above zk_host=$ZK_HOST @@ -848,15 +897,15 @@ else zk_port=$[$SOLR_PORT+1000] zk_host=localhost:$zk_port fi - + for (( s=1; s<$CLOUD_NUM_NODES; s++ )) do ndx=$[$s+1] next_port=${CLOUD_PORTS[$s]} echo -e "\n\nStarting node$ndx on port $next_port using command:\n" - echo -e "solr start -cloud -d node$ndx -p $next_port -z $zk_host \n\n" + echo -e "solr start -cloud -d node$ndx -p $next_port -z $zk_host $DASHM $DASHA \n\n" # call this script again with correct args for next node - $SOLR_TIP/bin/solr start -cloud -d node$ndx -p $next_port -z $zk_host + $SOLR_TIP/bin/solr start -cloud -d node$ndx -p $next_port -z $zk_host $DASHM $DASHA done # TODO: better (shorter) name?? diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 08e0b2c52bb..3eb217320a1 100644 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -30,8 +30,16 @@ REM Used to report errors before exiting the script set SCRIPT_ERROR= set NO_USER_PROMPT=0 +REM Allow user to import vars from an include file +REM vars set in the include file can be overridden with +REM command line args +IF "%SOLR_INCLUDE%"=="" set SOLR_INCLUDE=solr.in.cmd +IF EXIST "%SOLR_INCLUDE%" CALL "%SOLR_INCLUDE%" + REM Verify Java is available -if NOT DEFINED JAVA_HOME goto need_java_home +IF DEFINED SOLR_JAVA_HOME set "JAVA_HOME=%SOLR_JAVA_HOME%" +IF NOT DEFINED JAVA_HOME goto need_java_home +set JAVA_HOME=%JAVA_HOME:"=% "%JAVA_HOME%"\bin\java -version:1.8 -version > nul 2>&1 IF ERRORLEVEL 1 "%JAVA_HOME%"\bin\java -version:1.7 -version > nul 2>&1 IF ERRORLEVEL 1 goto need_java_vers @@ -59,11 +67,11 @@ IF "%1"=="start" goto set_script_cmd IF "%1"=="stop" goto set_script_cmd IF "%1"=="restart" goto set_script_cmd IF "%1"=="healthcheck" ( -REM healthcheck uses different arg parsing strategy -SHIFT -goto parse_healthcheck_args + REM healthcheck uses different arg parsing strategy + SHIFT + goto parse_healthcheck_args ) -goto include_vars +goto parse_args :usage IF NOT "%SCRIPT_ERROR%"=="" ECHO %SCRIPT_ERROR% @@ -81,18 +89,22 @@ goto done @echo Usage: solr COMMAND OPTIONS @echo where COMMAND is one of: start, stop, restart, healthcheck @echo. -@echo Example: Start Solr running in the background on port 8984: +@echo Standalone server example (start Solr running in the background on port 8984): @echo. -@echo ./solr start -p 8984 +@echo solr start -p 8984 +@echo. +@echo SolrCloud example (start Solr running in SolrCloud mode using localhost:2181 to connect to ZooKeeper, with 1g max heap size and remote Java debug options enabled): +@echo. +@echo solr start -c -m 1g -z localhost:2181 -a "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1044" @echo. @echo Pass -help after any COMMAND to see command-specific usage information, -@echo such as: ./solr start -help +@echo such as: solr start -help or solr stop -help @echo. goto done :start_usage @echo. -@echo Usage: solr %SCRIPT_CMD% [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-a "additional-options"] [-V] +@echo Usage: solr %SCRIPT_CMD% [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-a "additional-options"] [-V] @echo. @echo -f Start Solr in foreground; default starts Solr in the background @echo and sends stdout / stderr to solr-PORT-console.log @@ -112,6 +124,13 @@ goto done @echo -m memory Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g @echo results in: -Xms4g -Xmx4g; by default, this script sets the heap size to 512m @echo. +@echo -s dir Sets the solr.solr.home system property; Solr will create core directories under +@echo this directory. This allows you to run multiple Solr instances on the same host +@echo while reusing the same server directory set using the -d parameter. If set, the +@echo specified directory should contain a solr.xml file. The default value is example/solr. +@echo This parameter is ignored when running examples (-e), as the solr.solr.home depends +@echo on which example is run. +@echo. @echo -e example Name of the example to run; available examples: @echo cloud: SolrCloud example @echo default: Solr default example @@ -154,14 +173,6 @@ goto done @echo. goto done -REM Allow user to import vars from an include file -REM vars set in the include file can be overridden with -REM command line args -:include_vars -IF "%SOLR_INCLUDE%"=="" set SOLR_INCLUDE=solr.in.cmd -IF EXIST "%SOLR_INCLUDE%" CALL "%SOLR_INCLUDE%" -goto parse_args - REM Really basic command-line arg parsing :parse_args IF "%SCRIPT_CMD%"=="" set SCRIPT_CMD=start @@ -177,6 +188,8 @@ IF "%1"=="-c" goto set_cloud_mode IF "%1"=="-cloud" goto set_cloud_mode IF "%1"=="-d" goto set_server_dir IF "%1"=="-dir" goto set_server_dir +IF "%1"=="-s" goto set_solr_home_dir +IF "%1"=="-solr.home" goto set_solr_home_dir IF "%1"=="-e" goto set_example IF "%1"=="-example" goto set_example IF "%1"=="-h" goto set_host @@ -190,13 +203,14 @@ IF "%1"=="-zkhost" goto set_zookeeper IF "%1"=="-a" goto set_addl_opts IF "%1"=="-addlopts" goto set_addl_opts IF "%1"=="-noprompt" goto set_noprompt +IF "%1"=="-k" goto set_stop_key +IF "%1"=="-key" goto set_stop_key IF NOT "%1"=="" goto invalid_cmd_line -process_script_cmd :set_script_cmd set SCRIPT_CMD=%1 SHIFT -goto include_vars +goto parse_args :set_foreground_mode set FG=1 @@ -232,6 +246,19 @@ SHIFT SHIFT goto parse_args +:set_solr_home_dir + +set "arg=%~2" +set firstChar=%arg:~0,1% +IF "%firstChar%"=="-" ( + set SCRIPT_ERROR=Expected directory but found %2 instead! + goto invalid_cmd_line +) +set "SOLR_HOME=%~2" +SHIFT +SHIFT +goto parse_args + :set_example set "arg=%~2" @@ -256,7 +283,6 @@ IF "%firstChar%"=="-" ( ) set SOLR_HEAP=%~2 -@echo SOLR_HEAP=%SOLR_HEAP% SHIFT SHIFT goto parse_args @@ -287,6 +313,19 @@ SHIFT SHIFT goto parse_args +:set_stop_key +set "arg=%~2" +set firstChar=%arg:~0,1% +IF "%firstChar%"=="-" ( + set SCRIPT_ERROR=Expected port but found %2 instead! + goto invalid_cmd_line +) + +set STOP_KEY=%~2 +SHIFT +SHIFT +goto parse_args + :set_zookeeper set "arg=%~2" @@ -302,7 +341,6 @@ SHIFT goto parse_args :set_addl_opts - set "arg=%~2" set "SOLR_ADDL_ARGS=%~2" SHIFT @@ -320,7 +358,7 @@ REM Perform the requested command after processing args IF "%verbose%"=="1" ( @echo Using Solr root directory: %SOLR_TIP% @echo Using Java: %JAVA% - %JAVA% -version + "%JAVA%" -version ) IF NOT "%SOLR_HOST%"=="" ( @@ -359,6 +397,19 @@ IF "%EXAMPLE%"=="" ( :start_solr IF "%SOLR_HOME%"=="" set "SOLR_HOME=%SOLR_SERVER_DIR%\solr" +IF NOT EXIST "%SOLR_HOME%\" ( + IF EXIST "%SOLR_SERVER_DIR%\%SOLR_HOME%" ( + set "SOLR_HOME=%SOLR_SERVER_DIR%\%SOLR_HOME%" + ) ELSE ( + set SCRIPT_ERROR=Solr home directory %SOLR_HOME% not found! + goto err + ) +) + +IF NOT EXIST "%SOLR_HOME%\solr.xml" ( + set SCRIPT_ERROR=Solr home directory %SOLR_HOME% must contain solr.xml! + goto err +) IF "%STOP_KEY%"=="" set STOP_KEY=solrrocks @@ -541,7 +592,13 @@ for /l %%x in (1, 1, !CLOUD_NUM_NODES!) do ( @echo Cloning %DEFAULT_SERVER_DIR% into %SOLR_TIP%\node%%x xcopy /Q /E /I "%DEFAULT_SERVER_DIR%" "%SOLR_TIP%\node%%x" ) - + + IF NOT "!SOLR_HEAP!"=="" ( + set "DASHM=-m !SOLR_HEAP!" + ) ELSE ( + set "DASHM=" + ) + IF %%x EQU 1 ( set EXAMPLE= IF NOT "!ZK_HOST!"=="" ( @@ -550,8 +607,8 @@ for /l %%x in (1, 1, !CLOUD_NUM_NODES!) do ( set "DASHZ=" ) @echo Starting node1 on port !NODE_PORT! using command: - @echo solr -cloud -p !NODE_PORT! -d node1 !DASHZ! - START "" "%SDIR%\solr" -f -cloud -p !NODE_PORT! -d node1 !DASHZ! + @echo solr -cloud -p !NODE_PORT! -d node1 !DASHZ! !DASHM! + START "" "%SDIR%\solr" -f -cloud -p !NODE_PORT! -d node1 !DASHZ! !DASHM! set NODE1_PORT=!NODE_PORT! ) ELSE ( IF "!ZK_HOST!"=="" ( @@ -559,8 +616,8 @@ for /l %%x in (1, 1, !CLOUD_NUM_NODES!) do ( set "ZK_HOST=localhost:!ZK_PORT!" ) @echo Starting node%%x on port !NODE_PORT! using command: - @echo solr -cloud -p !NODE_PORT! -d node%%x -z !ZK_HOST! - START "" "%SDIR%\solr" -f -cloud -p !NODE_PORT! -d node%%x -z !ZK_HOST! + @echo solr -cloud -p !NODE_PORT! -d node%%x -z !ZK_HOST! !DASHM! + START "" "%SDIR%\solr" -f -cloud -p !NODE_PORT! -d node%%x -z !ZK_HOST! !DASHM! ) timeout /T 10 diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd index 76fef11703d..4724b279d75 100644 --- a/solr/bin/solr.in.cmd +++ b/solr/bin/solr.in.cmd @@ -25,7 +25,7 @@ REM Increase Java Min/Max Heap as needed to support your indexing / query needs set SOLR_JAVA_MEM=-Xms512m -Xmx512m -XX:MaxPermSize=256m -XX:PermSize=256m REM Enable verbose GC logging -set GC_LOG_OPTS=-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution +set GC_LOG_OPTS=-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime REM These GC settings have shown to work well for a number of common Solr workloads set GC_TUNE=-XX:-UseSuperWord ^ @@ -34,11 +34,13 @@ set GC_TUNE=-XX:-UseSuperWord ^ -XX:TargetSurvivorRatio=90 ^ -XX:MaxTenuringThreshold=8 ^ -XX:+UseConcMarkSweepGC ^ + -XX:+UseParNewGC ^ + -XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 ^ -XX:+CMSScavengeBeforeRemark ^ -XX:PretenureSizeThreshold=64m ^ -XX:CMSFullGCsBeforeCompaction=1 ^ -XX:+UseCMSInitiatingOccupancyOnly ^ - -XX:CMSInitiatingOccupancyFraction=70 ^ + -XX:CMSInitiatingOccupancyFraction=50 ^ -XX:CMSTriggerPermRatio=80 ^ -XX:CMSMaxAbortablePrecleanTime=6000 ^ -XX:+CMSParallelRemarkEnabled ^ diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh index ce41d1de827..e89e7900c08 100644 --- a/solr/bin/solr.in.sh +++ b/solr/bin/solr.in.sh @@ -23,7 +23,7 @@ SOLR_JAVA_MEM="-Xms512m -Xmx512m -XX:MaxPermSize=256m -XX:PermSize=256m" # Enable verbose GC logging GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \ --XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution" +-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime" # These GC settings have shown to work well for a number of common Solr workloads GC_TUNE="-XX:-UseSuperWord \ @@ -32,11 +32,13 @@ GC_TUNE="-XX:-UseSuperWord \ -XX:TargetSurvivorRatio=90 \ -XX:MaxTenuringThreshold=8 \ -XX:+UseConcMarkSweepGC \ +-XX:+UseParNewGC \ +-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \ -XX:+CMSScavengeBeforeRemark \ -XX:PretenureSizeThreshold=64m \ -XX:CMSFullGCsBeforeCompaction=1 \ -XX:+UseCMSInitiatingOccupancyOnly \ --XX:CMSInitiatingOccupancyFraction=70 \ +-XX:CMSInitiatingOccupancyFraction=50 \ -XX:CMSTriggerPermRatio=80 \ -XX:CMSMaxAbortablePrecleanTime=6000 \ -XX:+CMSParallelRemarkEnabled \