Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-6313: Improve SolrCloud cloud-dev scripts. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1616275 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2014-08-06 16:46:31 +00:00
parent 6662a12c71
commit ddbd55d24d
7 changed files with 113 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -330,6 +330,8 @@ Other Changes
* SOLR-5664: Added support for multi-valued field highlighting in /browse UI. (ehatcher)
* SOLR-6313: Improve SolrCloud cloud-dev scripts. (Mark Miller, Vamsee Yarlagadda)
================== 4.9.0 ==================
Versions of Major Components

2
solr/cloud-dev/control.sh
View File

@ -4,7 +4,7 @@ source ./functions.sh
case "$1" in
start)
start $2 $3
start $2 $3 "$4"
;;
stop)
stop $2

17
solr/cloud-dev/functions.sh
View File

@ -1,7 +1,7 @@
JAVA_OPTS="-server -Xms256M -Xmx256M"
BASE_PORT=7572
BASE_STOP_PORT=6572
ZK_PORT="9983"
INT_JAVA_OPTS="-server -Xms256M -Xmx256M"
BASE_PORT=8900
BASE_STOP_PORT=9900
ZK_PORT="2414"
rebuild() {
echo "Rebuilding"
@ -14,13 +14,8 @@ rebuild() {
}
setports() {
if [ "1" = "$1" ]; then
PORT="8983"
STOP_PORT="7983"
else
PORT="$(( $BASE_PORT + $1 ))"
STOP_PORT="$(( $BASE_STOP_PORT + $1 ))"
fi
PORT="$(( $BASE_PORT + $1 ))"
STOP_PORT="$(( $BASE_STOP_PORT + $1 ))"
}
reinstall() {

21
solr/cloud-dev/solrcloud-start-existing.sh
View File

@ -1,29 +1,34 @@
#!/bin/bash
numServers=$1
baseJettyPort=7572
baseStopPort=6572
baseJettyPort=8900
baseStopPort=9900
die () {
echo >&2 "$@"
exit 1
}
[ "$#" -eq 1 ] || die "1 argument required, $# provided, usage: solrcloud-start-exisiting.sh {numServers}"
[ "$#" -eq 1 ] || die "1 argument required, $# provided, usage: solrcloud-start-exisiting.sh [numServers]"
cd ..
cd example1
echo "starting example1"
java -DzkRun -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example1.log 2>&1 &
cd examplezk
stopPort=1313
jettyPort=8900
exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 &
# TODO: we could also remove the default core
cd ..
cd example
for (( i=2; i <= $numServers; i++ ))
for (( i=1; i <= $numServers; i++ ))
do
echo "starting example$i"
cd ../example$i
stopPort=`expr $baseStopPort + $i`
jettyPort=`expr $baseJettyPort + $i`
java -Xmx1g -Djetty.port=$jettyPort -DzkHost=localhost:9983 -DnumShards=1 -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
exec -a jetty java -Xmx1g $JAVA_OPTS -Djetty.port=$jettyPort -DzkHost=localhost:9900 -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
done

35
solr/cloud-dev/solrcloud-start.sh
View File

@ -1,19 +1,22 @@
#!/bin/bash
# To run on hdfs, try something along the lines of:
# export JAVA_OPTS="-Dsolr.directoryFactory=solr.HdfsDirectoryFactory -Dsolr.lock.type=hdfs -Dsolr.hdfs.home=hdfs://localhost:8020/solr -Dsolr.hdfs.confdir=/etc/hadoop_conf/conf"
numServers=$1
numShards=$2
baseJettyPort=7572
baseStopPort=6572
baseJettyPort=8900
baseStopPort=9900
zkaddress = localhost:2181/solr
zkAddress=localhost:9900
die () {
echo >&2 "$@"
exit 1
}
[ "$#" -eq 2 ] || die "2 arguments required, $# provided, usage: solrcloud-start.sh {numServers} {numShards}"
[ "$#" -eq 2 ] || die "2 arguments required, $# provided, usage: solrcloud-start.sh [numServers] [numShards]"
cd ..
@ -29,6 +32,7 @@ rm -r -f example/solr/zoo_data
rm -r -f example/solr/collection1/data
rm -f example/example.log
ant -f ../build.xml clean
ant example dist
rm -r example/solr-webapp/*
@ -40,23 +44,28 @@ do
cp -r -f example example$i
done
rm -r -f examplezk
cp -r -f example examplezk
cp core/src/test-files/solr/solr-no-core.xml examplezk/solr/solr.xml
rm -r -f examplezk/solr/collection1/core.properties
cd examplezk
stopPort=1313
jettyPort=8900
exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 &
cd ..
java -classpath "example1/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/*" org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost 127.0.0.1:9983 -solrhome example1/solr -runzk 8983
# upload config files
java -classpath "example1/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/*" org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost $zkAddress -solrhome example1/solr
echo "starting example1"
cd example
cd example1
java -Xmx1g -DzkRun -DnumShards=$numShards -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example1.log 2>&1 &
for (( i=2; i <= $numServers; i++ ))
for (( i=1; i <= $numServers; i++ ))
do
echo "starting example$i"
cd ../example$i
stopPort=`expr $baseStopPort + $i`
jettyPort=`expr $baseJettyPort + $i`
java -Xmx1g -Djetty.port=$jettyPort -DzkHost=localhost:9983 -DnumShards=1 -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
exec -a jetty java -Xmx1g $JAVA_OPTS -DnumShards=$numShards -Djetty.port=$jettyPort -DzkHost=$zkAddress -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
done

54
solr/cloud-dev/stop.sh
View File

@ -1,7 +1,8 @@
#!/bin/bash
numServers=$1
baseStopPort=6572
baseJettyPort=8900
baseStopPort=9900
die () {
echo >&2 "$@"
@ -12,13 +13,52 @@ die () {
cd ../example
java -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar --stop
for (( i=2; i <= $numServers; i++ ))
for (( i=1; i <= $numServers; i++ ))
do
echo "stopping example$i"
cd ../example$i
stopPort=`expr $baseStopPort + $i`
echo "stopping example$i, stop port is $stopPort"
cd ../example$i
java -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar --stop
done
mkdir ../example-lastlogs
for (( i=1; i <= $numServers; i++ ))
do
cd ../example$i
jettyPort=`expr $baseJettyPort + $i`
echo "Make sure jetty stops and wait for it: $jettyPort"
pid=`lsof -i:$jettyPort -sTCP:LISTEN -t`
echo "pid:$pid"
#kill $pid
#wait $pid
if [ ! -z "$pid" ]
then
while [ -e /proc/$pid ]; do sleep 1; done
fi
# save the last shutdown logs
echo "copy example$i.log to lastlogs"
cp -r -f example$i.log ../example-lastlogs/example-last$i.log
done
# stop zk runner
java -DSTOP.PORT=1313 -DSTOP.KEY=key -jar start.jar --stop
echo "wait for port to be available: $baseJettyPort"
pid=`lsof -i:$baseJettyPort -sTCP:LISTEN -t`
echo "pid:$pid"
#kill $pid
#wait $pid
if [ ! -z "$pid" ]
then
while [ -e /proc/$pid ]; do sleep 0.1; done
fi
nc -w 30 127.0.0.1 $baseJettyPort
sleep 5

26
solr/core/src/java/org/apache/solr/cloud/ZkController.java
View File

@ -178,6 +178,12 @@ public final class ZkController {
// keeps track of replicas that have been asked to recover by leaders running on this node
private Map<String,String> replicasInLeaderInitiatedRecovery = new HashMap<String,String>();
// This is an expert and unsupported development mode that does not create
// an Overseer or register a /live node. This let's you monitor the cluster
// and interact with zookeeper via the Solr admin UI on a node outside the cluster,
// and so one that will not be killed or stopped when testing. See developer cloud-scripts.
private boolean zkRunOnly = Boolean.getBoolean("zkRunOnly"); // expert
public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort,
String localHostContext, int leaderVoteWait, int leaderConflictResolveWait, boolean genericCoreNodeNames, final CurrentCoreDescriptorProvider registerOnReconnect)
throws InterruptedException, TimeoutException, IOException
@ -228,16 +234,19 @@ public final class ZkController {
registerAllCoresAsDown(registerOnReconnect, false);
ElectionContext context = new OverseerElectionContext(zkClient,
overseer, getNodeName());
if (!zkRunOnly) {
ElectionContext context = new OverseerElectionContext(zkClient,
overseer, getNodeName());
ElectionContext prevContext = overseerElector.getContext();
if (prevContext != null) {
prevContext.cancelElection();
ElectionContext prevContext = overseerElector.getContext();
if (prevContext != null) {
prevContext.cancelElection();
}
overseerElector.setup(context);
overseerElector.joinElection(context, true);
}
overseerElector.setup(context);
overseerElector.joinElection(context, true);
zkStateReader.createClusterStateWatchersAndUpdate();
// we have to register as live first to pick up docs in the buffer
@ -686,6 +695,9 @@ public final class ZkController {
private void createEphemeralLiveNode() throws KeeperException,
InterruptedException {
if (zkRunOnly) {
return;
}
String nodeName = getNodeName();
String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
log.info("Register node as live in ZooKeeper:" + nodePath);