From bf0cf89a0339b5286b205205c6d539cbbe96a775 Mon Sep 17 00:00:00 2001 From: William Au Date: Wed, 3 May 2006 13:40:05 +0000 Subject: [PATCH] initial version git-svn-id: https://svn.apache.org/repos/asf/incubator/solr/trunk@399300 13f79535-47bb-0310-9956-ffa450edef68 --- example/solr/bin/abc | 158 ++++++++++++++++++ example/solr/bin/abo | 158 ++++++++++++++++++ example/solr/bin/backup | 101 ++++++++++++ example/solr/bin/backupcleaner | 127 +++++++++++++++ example/solr/bin/commit | 112 +++++++++++++ example/solr/bin/optimize | 111 +++++++++++++ example/solr/bin/readercycle | 111 +++++++++++++ example/solr/bin/rsyncd-disable | 70 ++++++++ example/solr/bin/rsyncd-enable | 69 ++++++++ example/solr/bin/rsyncd-start | 138 ++++++++++++++++ example/solr/bin/rsyncd-stop | 98 +++++++++++ example/solr/bin/scripts-util | 81 ++++++++++ example/solr/bin/snapcleaner | 133 +++++++++++++++ example/solr/bin/snapinstaller | 159 ++++++++++++++++++ example/solr/bin/snappuller | 241 ++++++++++++++++++++++++++++ example/solr/bin/snappuller-disable | 70 ++++++++ example/solr/bin/snappuller-enable | 70 ++++++++ example/solr/bin/snapshooter | 102 ++++++++++++ 18 files changed, 2109 insertions(+) create mode 100755 example/solr/bin/abc create mode 100755 example/solr/bin/abo create mode 100755 example/solr/bin/backup create mode 100755 example/solr/bin/backupcleaner create mode 100755 example/solr/bin/commit create mode 100755 example/solr/bin/optimize create mode 100755 example/solr/bin/readercycle create mode 100755 example/solr/bin/rsyncd-disable create mode 100755 example/solr/bin/rsyncd-enable create mode 100755 example/solr/bin/rsyncd-start create mode 100755 example/solr/bin/rsyncd-stop create mode 100755 example/solr/bin/scripts-util create mode 100755 example/solr/bin/snapcleaner create mode 100755 example/solr/bin/snapinstaller create mode 100755 example/solr/bin/snappuller create mode 100755 example/solr/bin/snappuller-disable create mode 100755 example/solr/bin/snappuller-enable create mode 100755 example/solr/bin/snapshooter diff --git a/example/solr/bin/abc b/example/solr/bin/abc new file mode 100755 index 00000000000..cabaa0dc7f2 --- /dev/null +++ b/example/solr/bin/abc @@ -0,0 +1,158 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to make an Atomic Backup after Commit of +# a Solr Lucene collection. + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset solr_hostname solr_port data_dir webapp_name user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-h hostname] [-p port] [-d dir] [-w webappname] [-u username] [-v] + -h specify Solr hostname + -p specify Solr port number + -d specify directory holding index data + -w specify name of Solr webapp (defaults to solr) + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts h:p:d:w:u:v OPTION +do + case $OPTION in + h) + solr_hostname="$OPTARG" + ;; + p) + solr_port="$OPTARG" + ;; + d) + data_dir="$OPTARG" + ;; + w) + webapp_name="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${solr_port} ]] +then + echo "Solr port number missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +# use default hostname if not specified +if [[ -z ${solr_hostname} ]] +then + solr_hostname=localhost +fi + +# use default webapp name if not specified +if [[ -z ${webapp_name} ]] +then + webapp_name=solr +fi + +fixUser "$@" + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +logMessage sending commit to Solr server at port ${solr_port} +rs=`curl http://${solr_hostname}:${solr_port}/solr/update -s -d ""` +if [[ $? != 0 ]] +then + logMessage failed to connect to Solr server at port ${solr_port} + logMessage commit failed + logExit failed 1 +fi + +# check status of commit request +rc=`echo $rs|cut -f2 -d'"'` +if [[ $? != 0 ]] +then + logMessage commit request to Solr at port ${solr_port} failed: + logMessage $rs + logExit failed 2 +fi + +# successful commit creates a snapshot file synchronously +lastsnap=`ls -drt1 ${data_dir}/snapshot.* 2> /dev/null | tail -1 ` + +if [[ $lastsnap == "" ]] +then + logMessage commit did not create snapshot at port ${solr_port}, backup failed: + logExit failed 3 +fi + +name=backup.${lastsnap##*snapshot.} +temp=temp-${name} + +if [[ -d ${data_dir}/${name} ]] +then + logMessage backup directory ${data_dir}/${name} already exists + logExit aborted 1 +fi + +if [[ -d ${data_dir}/${temp} ]] +then + logMessage backingup of ${data_dir}/${name} in progress + logExit aborted 1 +fi +logMessage making backup ${data_dir}/${name} + +# clean up after INT/TERM +trap 'echo cleaning up, please wait ...;/bin/rm -rf ${data_dir}/${name} ${data_dir}/${temp};logExit aborted 13' INT TERM + +# make a backup using hard links into temporary location +# then move it into place atomically +cp -lr ${lastsnap} ${data_dir}/${temp} +mv ${data_dir}/${temp} ${data_dir}/${name} + +logExit ended 0 diff --git a/example/solr/bin/abo b/example/solr/bin/abo new file mode 100755 index 00000000000..79afb62c798 --- /dev/null +++ b/example/solr/bin/abo @@ -0,0 +1,158 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to make an Atomic Backup after Optimize of +# a Solr Lucene collection. + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset solr_hostname solr_port data_dir webapp_name user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-h hostname] [-p port] [-d dir] [-w webapp_name] [-u username] [-v] + -h specify Solr hostname + -p specify Solr port number + -d specify directory holding index data + -w specify name of Solr webapp (defaults to solr) + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts h:p:d:w:u:v OPTION +do + case $OPTION in + h) + solr_hostname="$OPTARG" + ;; + p) + solr_port="$OPTARG" + ;; + d) + data_dir="$OPTARG" + ;; + w) + webapp_name="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${solr_port} ]] +then + echo "Solr port number missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +# use default hostname if not specified +if [[ -z ${solr_hostname} ]] +then + solr_hostname=localhost +fi + +# use default webapp name if not specified +if [[ -z ${webapp_name} ]] +then + webapp_name=solr +fi + +fixUser "$@" + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +logMessage sending optimize to Solr server at port ${solr_port} +rs=`curl http://${solr_hostname}:${solr_port}/solr/update -s -d ""` +if [[ $? != 0 ]] +then + logMessage failed to connect to Solr server at port ${solr_port} + logMessage optimize failed + logExit failed 1 +fi + +# check status of optimize request +rc=`echo $rs|cut -f2 -d'"'` +if [[ $? != 0 ]] +then + logMessage optimize request to Solr at port ${solr_port} failed: + logMessage $rs + logExit failed 2 +fi + +# successful optimize creates a snapshot file synchronously +lastsnap=`ls -drt1 ${data_dir}/snapshot.* | tail -1 ` + +if [[ $lastsnap == "" ]] +then + logMessage commit did not create snapshot at port ${solr_port}, backup failed: + logExit failed 3 +fi + +name=backup.${lastsnap##*snapshot.} +temp=temp-${name} + +if [[ -d ${data_dir}/${name} ]] +then + logMessage backup directory ${data_dir}/${name} already exists + logExit aborted 1 +fi + +if [[ -d ${data_dir}/${temp} ]] +then + logMessage backingup of ${data_dir}/${name} in progress + logExit aborted 1 +fi +logMessage making backup ${data_dir}/${name} + +# clean up after INT/TERM +trap 'echo cleaning up, please wait ...;/bin/rm -rf ${data_dir}/${name} ${data_dir}/${temp};logExit aborted 13' INT TERM + +# make a backup using hard links into temporary location +# then move it into place atomically +cp -lr ${lastsnap} ${data_dir}/${temp} +mv ${data_dir}/${temp} ${data_dir}/${name} + +logExit ended 0 diff --git a/example/solr/bin/backup b/example/solr/bin/backup new file mode 100755 index 00000000000..1b56b597a45 --- /dev/null +++ b/example/solr/bin/backup @@ -0,0 +1,101 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to make a backup of a Solr Lucene collection. + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset data_dir user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-d dir] [-u username] [-v] + -d specify directory holding index data + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts d:u:v OPTION +do + case $OPTION in + d) + data_dir="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +fixUser "$@" + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +name=backup.`date +"%Y%m%d%H%M%S"` +temp=temp-${name} + +if [[ -d ${data_dir}/${name} ]] +then + logMessage backup directory ${data_dir}/${name} already exists + logExit aborted 1 +fi + +if [[ -d ${data_dir}/${temp} ]] +then + logMessage backingup of ${data_dir}/${name} in progress + logExit aborted 1 +fi + +# clean up after INT/TERM +trap 'echo cleaning up, please wait ...;/bin/rm -rf ${data_dir}/${name} ${data_dir}/${temp};logExit aborted 13' INT TERM + +logMessage making backup ${data_dir}/${name} + +# make a backup using hard links into temporary location +# then move it into place atomically +cp -lr ${data_dir}/index ${data_dir}/${temp} +mv ${data_dir}/${temp} ${data_dir}/${name} + +logExit ended 0 + diff --git a/example/solr/bin/backupcleaner b/example/solr/bin/backupcleaner new file mode 100755 index 00000000000..3298803d1aa --- /dev/null +++ b/example/solr/bin/backupcleaner @@ -0,0 +1,127 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to clean up backups of a Solr Lucene collection. + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset days num data_dir user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog -D | -N [-d dir] [-u username] [-v] + -D cleanup backups more than days old + -N keep the most recent number of backups and + cleanup up the remaining ones that are not being pulled + -d specify directory holding index data + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts D:N:d:u:v OPTION +do + case $OPTION in + D) + days="$OPTARG" + ;; + N) + num="$OPTARG" + ;; + d) + data_dir="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${days} && -z ${num} ]] +then + echo "$USAGE" + exit 1 +fi + +fixUser "$@" + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +function remove +{ + logMessage removing backup $1 + /bin/rm -rf $1 +} + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +# trap control-c +trap 'echo "caught INT/TERM, exiting now but partial cleanup may have already occured";logExit aborted 13' INT TERM + +if [[ -n ${days} ]] +then + logMessage cleaning up backups more than ${days} days old + for i in `find ${data_dir} -name "backup.*" -maxdepth 1 -mtime +${days} -print` + do + remove $i + done +elif [[ -n ${num} ]] +then + logMessage cleaning up all backups except for the most recent ${num} ones + unset backups count + backups=`ls -cd ${data_dir}/backup.* 2>/dev/null` + if [[ $? == 0 ]] + then + count=`echo $backups|wc -w` + startpos=`expr $num + 1` + if [[ $count -gt $num ]] + then + for i in `echo $backups|cut -f${startpos}- -d" "` + do + remove $i + done + fi + fi +fi + +logExit ended 0 + + diff --git a/example/solr/bin/commit b/example/solr/bin/commit new file mode 100755 index 00000000000..6f102d81da1 --- /dev/null +++ b/example/solr/bin/commit @@ -0,0 +1,112 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to force a commit of all changes since last commit +# for a Solr server + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset solr_hostname solr_port webapp_name user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-h hostname] [-p port] [-w webapp_name] [-u username] [-v] + -h specify Solr hostname + -p specify Solr port number + -w specify name of Solr webapp (defaults to solr) + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts h:p:w:u:v OPTION +do + case $OPTION in + h) + solr_hostname="$OPTARG" + ;; + p) + solr_port="$OPTARG" + ;; + w) + webapp_name="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${solr_port} ]] +then + echo "Solr port number missing in $confFile or command line." + echo "$USAGE" + + + exit 1 +fi + +# use default hostname if not specified +if [[ -z ${solr_hostname} ]] +then + solr_hostname=localhost +fi + +# use default webapp name if not specified +if [[ -z ${webapp_name} ]] +then + webapp_name=solr +fi + +fixUser "$@" + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +rs=`curl http://${solr_hostname}:${solr_port}/solr/update -s -d ""` +if [[ $? != 0 ]] +then + logMessage failed to connect to Solr server at port ${solr_port} + logMessage commit failed + logExit failed 1 +fi + +# check status of commit request +rc=`echo $rs|cut -f2 -d'"'` +if [[ $? != 0 ]] +then + logMessage commit request to Solr at port ${solr_port} failed: + logMessage $rs + logExit failed 2 +fi + +logExit ended 0 diff --git a/example/solr/bin/optimize b/example/solr/bin/optimize new file mode 100755 index 00000000000..1418833cb73 --- /dev/null +++ b/example/solr/bin/optimize @@ -0,0 +1,111 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to force a optimized commit of all changes since last commit +# for a Solr server + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset solr_hostname solr_port webapp_name user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-h hostname] [-p port] [-u username] [-v] + -h specify Solr hostname + -p specify Solr port number + -w specify name of Solr webapp (defaults to solr) + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +originalargs="$@" +while getopts h:p:w:u:v OPTION +do + case $OPTION in + h) + solr_hostname="$OPTARG" + ;; + p) + solr_port="$OPTARG" + ;; + w) + webapp_name="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${solr_port} ]] +then + echo "Solr port number missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +# use default hostname if not specified +if [[ -z ${solr_hostname} ]] +then + solr_hostname=localhost +fi + +# use default webapp name if not specified +if [[ -z ${webapp_name} ]] +then + webapp_name=solr +fi + +fixUser "$@" + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +rs=`curl http://${solr_hostname}:${solr_port}/solr/update -s -d ""` +if [[ $? != 0 ]] +then + logMessage failed to connect to Solr server at port ${solr_port} + logMessage optimize failed + logExit failed 1 +fi + +# check status of optimize request +rc=`echo $rs|cut -f2 -d'"'` +if [[ $? != 0 ]] +then + logMessage optimize request to Solr at port ${solr_port} failed: + logMessage $rs + logExit failed 2 +fi + +logExit ended 0 diff --git a/example/solr/bin/readercycle b/example/solr/bin/readercycle new file mode 100755 index 00000000000..9051de0e67c --- /dev/null +++ b/example/solr/bin/readercycle @@ -0,0 +1,111 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to force all old readers closed and a new reader to be opened +# for a Solr server + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset solr_hostname solr_port webapp_name user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-p hostname] [-p port] [-w webapp_name] [-u username] [-v] + -h specify Solr hostname + -p specify Solr port number + -w specify name of Solr webapp (defaults to solr) + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +originalargs="$@" +while getopts h:p:w:u:v OPTION +do + case $OPTION in + h) + solr_hostname="$OPTARG" + ;; + p) + solr_port="$OPTARG" + ;; + w) + webapp_name="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${solr_port} ]] +then + echo "Solr port number missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +# use default hostname if not specified +if [[ -z ${solr_hostname} ]] +then + solr_hostname=localhost +fi + +# use default webapp name if not specified +if [[ -z ${webapp_name} ]] +then + webapp_name=solr +fi + +fixUser "$@" + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +rs=`curl http://${solr_hostname}:${solr_port}/solr/update -s -d ""` +if [[ $? != 0 ]] +then + logMessage failed to connect to Solr server at port ${solr_port} + logMessage reader cycle failed + logExit failed 1 +fi + +# check status of commit request +rc=`echo $rs|cut -f2 -d'"'` +if [[ $? != 0 ]] +then + logMessage reader cycle request to Solr at port ${solr_port} failed: + logMessage $rs + logExit failed 2 +fi + +logExit ended 0 diff --git a/example/solr/bin/rsyncd-disable b/example/solr/bin/rsyncd-disable new file mode 100755 index 00000000000..4d6bb39d843 --- /dev/null +++ b/example/solr/bin/rsyncd-disable @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to disable rsyncd + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/rsyncd.log + +# define usage string +USAGE="\ +usage: $prog [-u username] [-v] + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts u:v OPTION +do + case $OPTION in + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +fixUser "$@" + +start=`date +"%s"` + +logMessage disabled by $oldwhoami +logMessage command: $0 $@ +name=${solr_root}/logs/rsyncd-enabled + +if [[ -f ${name} ]] +then + rm -f ${name} +else + logMessage rsyncd not currently enabled + logExit exited 1 +fi + +logExit ended 0 diff --git a/example/solr/bin/rsyncd-enable b/example/solr/bin/rsyncd-enable new file mode 100755 index 00000000000..488145879f4 --- /dev/null +++ b/example/solr/bin/rsyncd-enable @@ -0,0 +1,69 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to enable rsyncd + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +log=${solr_root}/logs/rsyncd.log + +# define usage string +USAGE="\ +usage: $prog [-u username] [-v] + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts u:v OPTION +do + case $OPTION in + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +fixUser "$@" + +start=`date +"%s"` + +logMessage enabled by $oldwhoami +logMessage command: $0 $@ +name=${solr_root}/logs/rsyncd-enabled + +if [[ -f ${name} ]] +then + logMessage rsyncd already currently enabled + logExit exited 1 +else + touch ${name} +fi + +logExit ended 0 diff --git a/example/solr/bin/rsyncd-start b/example/solr/bin/rsyncd-start new file mode 100755 index 00000000000..fb2cb418cbe --- /dev/null +++ b/example/solr/bin/rsyncd-start @@ -0,0 +1,138 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to start rsyncd on master Solr server + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset data_dir solr_port rsyncd_port user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/rsyncd.log + +# define usage string +USAGE="\ +usage: $prog [-d dir] [-p portnum] [-u username] [-v] + -d specify directory holding index data + -p specify rsyncd port number + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts d:p:u:v OPTION +do + case $OPTION in + d) + data_dir="$OPTARG" + ;; + p) + rsyncd_port="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +fixUser "$@" + +# try to determine rsyncd port number from $confFile if not specified on +# command line, default to solr_port+10000 +if [[ -z ${rsyncd_port} ]] +then + if [[ "${solr_port}" ]] + then + rsyncd_port=`expr 10000 + ${solr_port}` + else + echo "rsyncd port number missing in $confFile or command line." + echo "$USAGE" + exit 1 + fi +fi + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +if [[ ! -f ${solr_root}/logs/rsyncd-enabled ]] +then + logMessage rsyncd disabled + exit 1 +fi + +if \ + rsync rsync://localhost:${rsyncd_port} >/dev/null 2>&1 +then + logMessage "rsyncd already running at port ${rsyncd_port}" + exit 1 +fi + +# create conf/rsyncd.conf on the fly, creating solrlogs directory if needed +if [[ ! -d ${solr_root}/conf ]] +then + mkdir ${solr_root}/conf +fi +cat < ${solr_root}/conf/rsyncd.conf +#### rsyncd.conf file #### + +uid = $(whoami) +gid = $(whoami) +use chroot = no +list = no +pid file = ${solr_root}/logs/rsyncd.pid +log file = ${solr_root}/logs/rsyncd.log +[solr] + path = ${data_dir} + comment = Solr +EOF + +rsync --daemon --port=${rsyncd_port} --config=${solr_root}/conf/rsyncd.conf + +# first make sure rsyncd is accepting connections +i=1 +while \ + ! rsync rsync://localhost:${rsyncd_port} >/dev/null 2>&1 +do + if (( i++ > 15 )) + then + logMessage "rsyncd not accepting connections, exiting" >&2 + exit 2 + fi + sleep 1 +done + +logMessage rsyncd started with data_dir=${data_dir} and accepting requests diff --git a/example/solr/bin/rsyncd-stop b/example/solr/bin/rsyncd-stop new file mode 100755 index 00000000000..04177b371cd --- /dev/null +++ b/example/solr/bin/rsyncd-stop @@ -0,0 +1,98 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to stop rsyncd on master Solr server + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/rsyncd.log + +# define usage string +USAGE="\ +usage: $prog [-u username] [-v] + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts u:v OPTION +do + case $OPTION in + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +fixUser "$@" + +logMessage stopped by $oldwhoami +logMessage command: $0 $@ + +# look for pid file +if [[ ! -f ${solr_root}/logs/rsyncd.pid ]] +then + logMessage "missing rsyncd pid file ${solr_root}/logs/rsyncd.pid" + exit 2 +fi + +# get PID from file +pid=$(<${solr_root}/logs/rsyncd.pid) +if [[ -z $pid ]] +then + logMessage "unable to get rsyncd's PID" + exit 2 +fi + +kill $pid + +# wait until rsyncd dies or we time out +dead=0 +timer=0 +timeout=300 +while (( ! dead && timer < timeout )) +do + if ps -eo pid | grep -qw $pid + then + kill $pid + (( timer++ )) + sleep 1 + else + dead=1 + fi +done +if ps -eo pid | grep -qw $pid +then + logMessage rsyncd failed to stop after $timeout seconds + exit 3 +fi + +# remove rsyncd.conf +/bin/rm -f ${solr_root}/conf/rsyncd.conf \ No newline at end of file diff --git a/example/solr/bin/scripts-util b/example/solr/bin/scripts-util new file mode 100755 index 00000000000..6aa54471f46 --- /dev/null +++ b/example/solr/bin/scripts-util @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to disable rsyncd + +export PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH + +# set up variables +prog=${0##*/} + +# source the config file if present +confFile=${solr_root}/conf/scripts.conf +if [[ -f $confFile ]] +then + . $confFile +fi + +function fixUser +{ +# set user to $(whoami) if not specified + if [[ -z ${user} ]] + then + user=$(whoami) + fi + +# sudo + if [[ $(whoami) != ${user} ]] + then + sudo -u ${user} $0 "$@" + exit $? + fi + + oldwhoami=$(who -m | cut -d' ' -f1 | sed -e's/^.*!//') + + if [[ "${oldwhoami}" == "" ]] + then + oldwhoami=`ps h -Hfp $(pgrep -g0 ${0##*/}) | tail -1|cut -f1 -d" "` + fi +} + +function timeStamp +{ + date +'%Y/%m/%d %H:%M:%S' +} + +function logMessage +{ + echo $(timeStamp) $@>>$log + if [[ -n ${verbose} ]] + then + echo $@ + fi +} + +function logExit +{ + end=`date +"%s"` + diff=`expr $end - $start` + echo "$(timeStamp) $1 (elapsed time: $diff sec)">>$log + exit $2 +} + +# create logs directory if not there +if [[ ! -d ${solr_root}/logs ]] +then + mkdir ${solr_root}/logs +fi + diff --git a/example/solr/bin/snapcleaner b/example/solr/bin/snapcleaner new file mode 100755 index 00000000000..9b1acad3cfb --- /dev/null +++ b/example/solr/bin/snapcleaner @@ -0,0 +1,133 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to clean up snapshots of a Solr Lucene collection. + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset days num data_dir user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog -D | -N [-d dir] [-u username] [-v] + -D cleanup snapshots more than days old + -N keep the most recent number of snapshots and + cleanup up the remaining ones that are not being pulled + -d specify directory holding index data + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts D:N:d:u:v OPTION +do + case $OPTION in + D) + days="$OPTARG" + ;; + N) + num="$OPTARG" + ;; + d) + data_dir="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${days} && -z ${num} ]] +then + echo "$USAGE" + exit 1 +fi + +fixUser "$@" + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +function remove +{ + syncing=`ps -fwwwu ${user}|grep -w rsync|grep -v grep|grep -w $1` + if [[ -n $syncing ]] + then + logMessage $1 not removed - rsync in progress + else + logMessage removing snapshot $1 + /bin/rm -rf $1 + fi +} + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +# trap control-c +trap 'echo "caught INT/TERM, exiting now but partial cleanup may have already occured";logExit aborted 13' INT TERM + +if [[ -n ${days} ]] +then + logMessage cleaning up snapshots more than ${days} days old + for i in `find ${data_dir} -name "snapshot.*" -maxdepth 1 -mtime +${days} -print` + do + remove $i + done +elif [[ -n ${num} ]] +then + logMessage cleaning up all snapshots except for the most recent ${num} ones + unset snapshots count + snapshots=`ls -cd ${data_dir}/snapshot.* 2>/dev/null` + if [[ $? == 0 ]] + then + count=`echo $snapshots|wc -w` + startpos=`expr $num + 1` + if [[ $count -gt $num ]] + then + for i in `echo $snapshots|cut -f${startpos}- -d" "` + do + remove $i + done + fi + fi +fi + +logExit ended 0 + + diff --git a/example/solr/bin/snapinstaller b/example/solr/bin/snapinstaller new file mode 100755 index 00000000000..a4d4d7a2de0 --- /dev/null +++ b/example/solr/bin/snapinstaller @@ -0,0 +1,159 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to install a snapshot into place as the Lucene collection +# for a Solr server + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset master_host master_status_dir data_dir user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-M master] [-S sdir] [-d dir] [-u username] [-v] + -M master specify hostname of master server from where to pull index + snapshot + -S specify directory holding snapshot status on master server + -d specify directory holding index data on local machine + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts M:S:d:u:v OPTION +do + case $OPTION in + M) + master_host="$OPTARG" + ;; + S) + master_status_dir="$OPTARG" + ;; + d) + data_dir="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${master_host} ]] +then + echo "name of master server missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +if [[ -z ${master_status_dir} ]] +then + echo "directory holding snapshot status on master server missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +fixUser "$@" + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +# assume relative path to start at ${solr_root} +if [[ "`echo ${master_status_dir}|cut -c1`" != "/" ]] +then + master_status_dir=${solr_root}/${master_status_dir} +fi + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +# get directory name of latest snapshot +name=`ls -d ${data_dir}/snapshot.* 2>/dev/null|grep -v wip|sort -r|head -1` + +# clean up after INT/TERM +trap 'echo "caught INT/TERM, exiting now but partial installation may have already occured";/bin/rm -rf ${data_dir"/index.tmp$$;logExit aborted 13' INT TERM + +# is there a snapshot +if [[ "${name}" == "" ]] +then + logMessage no shapshot available + logExit ended 0 +fi + +# has snapshot already been installed +if [[ ${name} == `cat ${solr_root}/logs/snapshot.current 2>/dev/null` ]] +then + logMessage latest snapshot ${name} already installed + logExit ended 0 +fi + +# make sure master has directory for hold slaves stats/state +if + ! ssh -o StrictHostKeyChecking=no ${master_host} mkdir -p ${master_status_dir} +then + logMessage failed to ssh to master ${master_host}, snapshot status not updated on master +fi + +# install using hard links into temporary directory +# remove original index and then atomically copy new one into place +logMessage installing snapshot ${name} +cp -lr ${name}/ ${data_dir}/index.tmp$$ +/bin/rm -rf ${data_dir}/index +mv -f ${data_dir}/index.tmp$$ ${data_dir}/index + +# update distribution stats +echo ${name} > ${solr_root}/logs/snapshot.current + +# push stats/state to master +if + ! scp -q -o StrictHostKeyChecking=no ${solr_root}/logs/snapshot.current ${master_host}:${master_status_dir}/snapshot.current.`uname -n` +then + logMessage failed to ssh to master ${master_host}, snapshot status not updated on master +fi + +# notify Solr to open a new Searcher +logMessage notifing Solr to open a new Searcher +${solr_root}/bin/commit +if [[ $? != 0 ]] +then + logMessage failed to connect to Solr server + logMessage snapshot installed but Solr server has not open a new Searcher + logExit failed 1 +fi + +logExit ended 0 diff --git a/example/solr/bin/snappuller b/example/solr/bin/snappuller new file mode 100755 index 00000000000..62f938f3bcf --- /dev/null +++ b/example/solr/bin/snappuller @@ -0,0 +1,241 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to copy snapshots of a Solr Lucene collection from the master + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset master_host rsyncd_port master_data_dir master_status_dir snap_name +unset sizeonly stats data_dir user verbose compress startStatus +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-M master] [-P portnum] [-D mdir] [-S sdir] [-n snapshot] [-d dir] [-u username] [-svz] + -M master specify hostname of master server from where to pull index + snapshot + -P port specify rsyncd port number of master server from where to + pull index snapshot + -D specify directory holding index data on master server + -S specify directory holding snapshot status on master server + -n snapshot pull a specific snapshot by name + -d specify directory holding index data on local machine + -u specify user to sudo to before running script + -s use the --size-only option with rsync + -v increase verbosity (-vv show file transfer stats also) + -z enable compression of data +" + +# parse args +while getopts M:P:D:S:n:d:u:svz OPTION +do + case $OPTION in + M) + master_host="$OPTARG" + ;; + P) + rsyncd_port="$OPTARG" + ;; + D) + master_data_dir="$OPTARG" + ;; + S) + master_status_dir="$OPTARG" + ;; + n) + snap_name="$OPTARG" + ;; + d) + data_dir="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + s) + sizeonly="--size-only" + ;; + v) + [[ -n $verbose ]] && stats="--stats" || verbose=v + ;; + z) + compress="z" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +if [[ -z ${master_host} ]] +then + echo "name of master server missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +# try to determine rsyncd port number from $confFile if not specified on +# command line, default to solr_port+10000 +if [[ -z ${rsyncd_port} ]] +then + if [[ "${solr_port}" ]] + then + rsyncd_port=`expr 10000 + ${solr_port}` + else + echo "rsyncd port number of master server missing in $confFile or command line." + echo "$USAGE" + exit 1 + fi +fi + +if [[ -z ${master_data_dir} ]] +then + echo "directory holding index data on master server missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +if [[ -z ${master_status_dir} ]] +then + echo "directory holding snapshot status on master server missing in $confFile or command line." + echo "$USAGE" + exit 1 +fi + +fixUser "$@" + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +# assume relative path to start at ${solr_root} +if [[ "`echo ${master_data_dir}|cut -c1`" != "/" ]] +then + master_data_dir=${solr_root}/${master_data_dir} +fi +if [[ "`echo ${master_status_dir}|cut -c1`" != "/" ]] +then + master_status_dir=${solr_root}/${master_status_dir} +fi + +# push stats/state to master if necessary +function pushStatus +{ + scp -q -o StrictHostKeyChecking=no ${solr_root}/logs/snappuller.status ${master_host}:${master_status_dir}/snapshot.status.`uname -n` +} + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +if [[ ! -f ${solr_root}/logs/snappuller-enabled ]] +then + logMessage snappuller disabled + exit 1 +fi + +# make sure we can ssh to master +if + ! ssh -o StrictHostKeyChecking=no ${master_host} id 1>/dev/null 2>&1 +then + logMessage failed to ssh to master ${master_host} + exit 1 +fi + +# get directory name of latest snapshot if not specified on command line +if [[ -z ${snap_name} ]] +then + snap_name=`ssh -o StrictHostKeyChecking=no ${master_host} "ls -d ${master_data_dir}/snapshot.* 2>/dev/null"|tail -1` +fi +if [[ "${snap_name}" == "" ]] +then + logMessage no snapshot available on ${master_host} in ${master_data_dir} + logExit ended 0 +else + name=`basename ${snap_name}` +fi + +# clean up after INT/TERM +trap 'echo cleaning up, please wait ...;/bin/rm -rf ${data_dir}/${name} ${data_dir}/${name}-wip;echo ${startStatus} aborted:$(timeStamp)>${solr_root}/logs/snappuller.status;pushStatus;logExit aborted 13' INT TERM + +if [[ -d ${data_dir}/${name} || -d ${data_dir}/${name}-wip ]] +then + logMessage no new snapshot available on ${master_host} in ${master_data_dir} + logExit ended 0 +fi + +# take a snapshot of current index so that only modified files will be rsync-ed +# put the snapshot in the 'work-in-progress" directory to prevent it from +# being installed while the copying is still in progress +cp -lr ${data_dir}/index ${data_dir}/${name}-wip +# force rsync of segments and .del files since we are doing size-only +if [[ -n ${sizeonly} ]] +then + rm -f ${data_dir}/${name}-wip/segments + rm -f ${data_dir}/${name}-wip/*.del +fi + +logMessage pulling snapshot ${name} + +# make sure master has directory for hold slaves stats/state +ssh -o StrictHostKeyChecking=no ${master_host} mkdir -p ${master_status_dir} + +# start new distribution stats +rsyncStart=`date` +startTimestamp=`date -d "$rsyncStart" +'%Y%m%d-%H%M%S'` +rsyncStartSec=`date -d "$rsyncStart" +'%s'` +startStatus="rsync of `basename ${name}` started:$startTimestamp" +echo ${startStatus} > ${solr_root}/logs/snappuller.status +pushStatus + +# rsync over files that have changed +rsync -Wa${verbose}${compress} --delete ${sizeonly} \ +${stats} rsync://${master_host}:${rsyncd_port}/solr/${name}/ ${data_dir}/${name}-wip + +rc=$? +rsyncEnd=`date` +endTimestamp=`date -d "$rsyncEnd" +'%Y%m%d-%H%M%S'` +rsyncEndSec=`date -d "$rsyncEnd" +'%s'` +elapsed=`expr $rsyncEndSec - $rsyncStartSec` +if [[ $rc != 0 ]] +then + logMessage rsync failed + /bin/rm -rf ${data_dir}/${name}-wip + echo ${startStatus} failed:$endTimestamp > ${solr_root}/logs/snappuller.status + pushStatus + logExit failed 1 +fi + +# move into place atomically +mv ${data_dir}/${name}-wip ${data_dir}/${name} + +# finish new distribution stats` +echo ${startStatus} ended:$endTimestamp rsync-elapsed:${elapsed} > ${solr_root}/logs/snappuller.status +pushStatus +logExit ended 0 diff --git a/example/solr/bin/snappuller-disable b/example/solr/bin/snappuller-disable new file mode 100755 index 00000000000..aac98e66ee3 --- /dev/null +++ b/example/solr/bin/snappuller-disable @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to disable snappuller + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/snappuller.log + +# define usage string +USAGE="\ +usage: $prog [-u username] [-v] + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts u:v OPTION +do + case $OPTION in + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +fixUser "$@" + +start=`date +"%s"` + +logMessage disabled by $oldwhoami +logMessage command: $0 $@ +name=${solr_root}/logs/snappuller-enabled + +if [[ -f ${name} ]] +then + rm -f ${name} +else + logMessage snappuller not currently enabled + logExit exited 1 +fi + +logExit ended 0 diff --git a/example/solr/bin/snappuller-enable b/example/solr/bin/snappuller-enable new file mode 100755 index 00000000000..9969f2763fc --- /dev/null +++ b/example/solr/bin/snappuller-enable @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to enable snappuller + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/snappuller.log + +# define usage string +USAGE="\ +usage: $prog [-u username] [-v] + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts u:v OPTION +do + case $OPTION in + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +fixUser "$@" + +start=`date +"%s"` + +logMessage enabled by $oldwhoami +logMessage command: $0 $@ +name=${solr_root}/logs/snappuller-enabled + +if [[ -f ${name} ]] +then + logMessage snappuller already currently enabled + logExit exited 1 +else + touch ${name} +fi + +logExit ended 0 diff --git a/example/solr/bin/snapshooter b/example/solr/bin/snapshooter new file mode 100755 index 00000000000..172d55b8829 --- /dev/null +++ b/example/solr/bin/snapshooter @@ -0,0 +1,102 @@ +#!/bin/bash +# +# Copyright 2006 The Apache Software Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Shell script to take a snapshot of a Solr Lucene collection. + +orig_dir=$(pwd) +cd ${0%/*}/.. +solr_root=$(pwd) +cd ${orig_dir} + +unset data_dir user verbose +. ${solr_root}/bin/scripts-util + +# set up variables +prog=${0##*/} +log=${solr_root}/logs/${prog}.log + +# define usage string +USAGE="\ +usage: $prog [-d dir] [-u username] [-v] + -d specify directory holding index data + -u specify user to sudo to before running script + -v increase verbosity +" + +# parse args +while getopts d:u:v OPTION +do + case $OPTION in + d) + data_dir="$OPTARG" + ;; + u) + user="$OPTARG" + ;; + v) + verbose="v" + ;; + *) + echo "$USAGE" + exit 1 + esac +done + +fixUser "$@" + +# use default value for data_dir if not specified +# relative path starts at ${solr_root} +if [[ -z ${data_dir} ]] +then + data_dir=${solr_root}/data +elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]] +then + data_dir=${solr_root}/${data_dir} +fi + +start=`date +"%s"` + +logMessage started by $oldwhoami +logMessage command: $0 $@ + +snap_name=snapshot.`date +"%Y%m%d%H%M%S"` +name=${data_dir}/${snap_name} +temp=${data_dir}/temp-${snap_name} + +if [[ -d ${name} ]] +then + logMessage snapshot directory ${name} already exists + logExit aborted 1 +fi + +if [[ -d ${temp} ]] +then + logMessage snapshoting of ${name} in progress + logExit aborted 1 +fi + +# clean up after INT/TERM +trap 'echo cleaning up, please wait ...;/bin/rm -rf ${name} ${temp};logExit aborted 13' INT TERM + +logMessage taking snapshot ${name} + +# take a snapshot using hard links into temporary location +# then move it into place atomically +cp -lr ${data_dir}/index ${temp} +mv ${temp} ${name} + +logExit ended 0 +