Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4580: Support for protecting content in ZooKeeper. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1621294 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2014-08-29 12:30:53 +00:00
parent 3d14ee5ae2
commit a61a530190
31 changed files with 1286 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -117,6 +117,8 @@ New Features
* SOLR-6403: TransactionLog replay status logging. (Mark Miller) * SOLR-6403: TransactionLog replay status logging. (Mark Miller)
* SOLR-4580: Support for protecting content in ZooKeeper. (Per Steffensen, Mark Miller)
Bug Fixes Bug Fixes
---------------------- ----------------------

20
solr/cloud-dev/clean.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
numServers=$1
die () {
echo >&2 "$@"
exit 1
}
[ "$#" -eq 1 ] || die "1 argument required, $# provided, usage: clean.sh {numServers}"
cd ..
for (( i=1; i <= $numServers; i++ ))
do
rm -r -f example$i
done
rm -r -f examplezk
rm -r -f example-lastlogs

11
solr/cloud-dev/functions.sh
View File

@ -2,6 +2,7 @@ INT_JAVA_OPTS="-server -Xms256M -Xmx256M"
BASE_PORT=8900 BASE_PORT=8900
BASE_STOP_PORT=9900 BASE_STOP_PORT=9900
ZK_PORT="2414" ZK_PORT="2414"
ZK_CHROOT="solr"
rebuild() { rebuild() {
echo "Rebuilding" echo "Rebuilding"
@ -26,17 +27,11 @@ reinstall() {
} }
start() { start() {
OPT="-DzkHost=localhost:$ZK_PORT -DzkRun" OPT="-DzkHost=localhost:$ZK_PORT/$ZK_CHROOT"
NUMSHARDS=$2 NUMSHARDS=$2
echo "Starting instance $1" echo "Starting instance $1"
if [ "1" = "$1" ]; then
if [ "" = "$NUMSHARDS" ]; then
NUMSHARDS="1"
fi
echo "Instance is running zk, numshards=$NUMSHARDS"
OPT="-DzkRun -Dbootstrap_conf=true -DnumShards=$NUMSHARDS"
fi
setports $1 setports $1
cd ../example$1 cd ../example$1
java $JAVA_OPTS -Djetty.port=$PORT $OPT -DSTOP.PORT=$STOP_PORT -DSTOP.KEY=key -jar start.jar 1>example$1.log 2>&1 & java $JAVA_OPTS -Djetty.port=$PORT $OPT -DSTOP.PORT=$STOP_PORT -DSTOP.KEY=key -jar start.jar 1>example$1.log 2>&1 &

6
solr/cloud-dev/solrcloud-start-existing.sh
View File

@ -5,6 +5,8 @@ numServers=$1
baseJettyPort=8900 baseJettyPort=8900
baseStopPort=9900 baseStopPort=9900
ZK_CHROOT="solr"
die () { die () {
echo >&2 "$@" echo >&2 "$@"
exit 1 exit 1
@ -18,7 +20,7 @@ cd ..
cd examplezk cd examplezk
stopPort=1313 stopPort=1313
jettyPort=8900 jettyPort=8900
exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 & exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkHost=localhost:9900/$ZK_CHROOT -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 &
# TODO: we could also remove the default core # TODO: we could also remove the default core
cd .. cd ..
@ -30,5 +32,5 @@ do
cd ../example$i cd ../example$i
stopPort=`expr $baseStopPort + $i` stopPort=`expr $baseStopPort + $i`
jettyPort=`expr $baseJettyPort + $i` jettyPort=`expr $baseJettyPort + $i`
exec -a jetty java -Xmx1g $JAVA_OPTS -Djetty.port=$jettyPort -DzkHost=localhost:9900 -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 & exec -a jetty java -Xmx1g $JAVA_OPTS -Djetty.port=$jettyPort -DzkHost=localhost:9900/$ZK_CHROOT -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
done done

10
solr/cloud-dev/solrcloud-start.sh
View File

@ -3,13 +3,16 @@
# To run on hdfs, try something along the lines of: # To run on hdfs, try something along the lines of:
# export JAVA_OPTS="-Dsolr.directoryFactory=solr.HdfsDirectoryFactory -Dsolr.lock.type=hdfs -Dsolr.hdfs.home=hdfs://localhost:8020/solr -Dsolr.hdfs.confdir=/etc/hadoop_conf/conf" # export JAVA_OPTS="-Dsolr.directoryFactory=solr.HdfsDirectoryFactory -Dsolr.lock.type=hdfs -Dsolr.hdfs.home=hdfs://localhost:8020/solr -Dsolr.hdfs.confdir=/etc/hadoop_conf/conf"
# To use ZooKeeper security, try:
# export JAVA_OPTS="-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider -DzkDigestUsername=admin-user -DzkDigestPassword=admin-password -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=readonly-password"
numServers=$1 numServers=$1
numShards=$2 numShards=$2
baseJettyPort=8900 baseJettyPort=8900
baseStopPort=9900 baseStopPort=9900
zkAddress=localhost:9900 zkAddress=localhost:9900/solr
die () { die () {
echo >&2 "$@" echo >&2 "$@"
@ -44,6 +47,7 @@ do
cp -r -f example example$i cp -r -f example example$i
done done
rm -r -f examplezk rm -r -f examplezk
cp -r -f example examplezk cp -r -f example examplezk
cp core/src/test-files/solr/solr-no-core.xml examplezk/solr/solr.xml cp core/src/test-files/solr/solr-no-core.xml examplezk/solr/solr.xml
@ -51,11 +55,11 @@ rm -r -f examplezk/solr/collection1/core.properties
cd examplezk cd examplezk
stopPort=1313 stopPort=1313
jettyPort=8900 jettyPort=8900
exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 & exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun=localhost:9900/solr -DzkHost=$zkAddress -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 &
cd .. cd ..
# upload config files # upload config files
java -classpath "example1/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/*" org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost $zkAddress -solrhome example1/solr java -classpath "example1/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/*" $JAVA_OPTS org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost $zkAddress -solrhome example1/solr
cd example cd example

8
solr/core/src/java/org/apache/solr/cloud/DistributedMap.java
View File

@ -47,7 +47,6 @@ public class DistributedMap {
private final String dir; private final String dir;
private SolrZkClient zookeeper; private SolrZkClient zookeeper;
private List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE;
private final String prefix = "mn-"; private final String prefix = "mn-";
@ -66,9 +65,6 @@ public class DistributedMap {
throw new SolrException(ErrorCode.SERVER_ERROR, e); throw new SolrException(ErrorCode.SERVER_ERROR, e);
} }
if (acl != null) {
this.acl = acl;
}
this.zookeeper = zookeeper; this.zookeeper = zookeeper;
} }
@ -113,10 +109,10 @@ public class DistributedMap {
throws KeeperException, InterruptedException { throws KeeperException, InterruptedException {
for (;;) { for (;;) {
try { try {
return zookeeper.create(path, data, acl, mode, true); return zookeeper.create(path, data, mode, true);
} catch (KeeperException.NoNodeException e) { } catch (KeeperException.NoNodeException e) {
try { try {
zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT, true); zookeeper.create(dir, new byte[0], CreateMode.PERSISTENT, true);
} catch (KeeperException.NodeExistsException ne) { } catch (KeeperException.NodeExistsException ne) {
// someone created it // someone created it
} }

34
solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java
View File

@ -18,6 +18,14 @@
package org.apache.solr.cloud; package org.apache.solr.cloud;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.TreeMap;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.SolrZkClient;
@ -28,19 +36,9 @@ import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher; import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.TreeMap;
/** /**
* A distributed queue from zk recipes. * A distributed queue from zk recipes.
*/ */
@ -53,7 +51,6 @@ public class DistributedQueue {
private final String dir; private final String dir;
private SolrZkClient zookeeper; private SolrZkClient zookeeper;
private List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE;
private final String prefix = "qn-"; private final String prefix = "qn-";
@ -61,11 +58,11 @@ public class DistributedQueue {
private final Overseer.Stats stats; private final Overseer.Stats stats;
public DistributedQueue(SolrZkClient zookeeper, String dir, List<ACL> acl) { public DistributedQueue(SolrZkClient zookeeper, String dir) {
this(zookeeper, dir, acl, new Overseer.Stats()); this(zookeeper, dir, new Overseer.Stats());
} }
public DistributedQueue(SolrZkClient zookeeper, String dir, List<ACL> acl, Overseer.Stats stats) { public DistributedQueue(SolrZkClient zookeeper, String dir, Overseer.Stats stats) {
this.dir = dir; this.dir = dir;
ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zookeeper.getZkClientTimeout()); ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zookeeper.getZkClientTimeout());
@ -78,9 +75,6 @@ public class DistributedQueue {
throw new SolrException(ErrorCode.SERVER_ERROR, e); throw new SolrException(ErrorCode.SERVER_ERROR, e);
} }
if (acl != null) {
this.acl = acl;
}
this.zookeeper = zookeeper; this.zookeeper = zookeeper;
this.stats = stats; this.stats = stats;
} }
@ -294,7 +288,7 @@ public class DistributedQueue {
children = orderedChildren(watcher); children = orderedChildren(watcher);
break; break;
} catch (KeeperException.NoNodeException e) { } catch (KeeperException.NoNodeException e) {
zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT, true); zookeeper.create(dir, new byte[0], CreateMode.PERSISTENT, true);
// go back to the loop and try again // go back to the loop and try again
} }
} }
@ -366,10 +360,10 @@ public class DistributedQueue {
throws KeeperException, InterruptedException { throws KeeperException, InterruptedException {
for (;;) { for (;;) {
try { try {
return zookeeper.create(path, data, acl, mode, true); return zookeeper.create(path, data, mode, true);
} catch (KeeperException.NoNodeException e) { } catch (KeeperException.NoNodeException e) {
try { try {
zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT, true); zookeeper.create(dir, new byte[0], CreateMode.PERSISTENT, true);
} catch (KeeperException.NodeExistsException ne) { } catch (KeeperException.NodeExistsException ne) {
// someone created it // someone created it
} }

6
solr/core/src/java/org/apache/solr/cloud/Overseer.java
View File

@ -1280,13 +1280,13 @@ public class Overseer implements Closeable {
static DistributedQueue getInQueue(final SolrZkClient zkClient, Stats zkStats) { static DistributedQueue getInQueue(final SolrZkClient zkClient, Stats zkStats) {
createOverseerNode(zkClient); createOverseerNode(zkClient);
return new DistributedQueue(zkClient, "/overseer/queue", null, zkStats); return new DistributedQueue(zkClient, "/overseer/queue", zkStats);
} }
/* Internal queue, not to be used outside of Overseer */ /* Internal queue, not to be used outside of Overseer */
static DistributedQueue getInternalQueue(final SolrZkClient zkClient, Stats zkStats) { static DistributedQueue getInternalQueue(final SolrZkClient zkClient, Stats zkStats) {
createOverseerNode(zkClient); createOverseerNode(zkClient);
return new DistributedQueue(zkClient, "/overseer/queue-work", null, zkStats); return new DistributedQueue(zkClient, "/overseer/queue-work", zkStats);
} }
/* Internal map for failed tasks, not to be used outside of the Overseer */ /* Internal map for failed tasks, not to be used outside of the Overseer */
@ -1314,7 +1314,7 @@ public class Overseer implements Closeable {
static DistributedQueue getCollectionQueue(final SolrZkClient zkClient, Stats zkStats) { static DistributedQueue getCollectionQueue(final SolrZkClient zkClient, Stats zkStats) {
createOverseerNode(zkClient); createOverseerNode(zkClient);
return new DistributedQueue(zkClient, "/overseer/collection-queue-work", null, zkStats); return new DistributedQueue(zkClient, "/overseer/collection-queue-work", zkStats);
} }
private static void createOverseerNode(final SolrZkClient zkClient) { private static void createOverseerNode(final SolrZkClient zkClient) {

7
solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
View File

@ -249,14 +249,12 @@ public class ZkCLI {
} }
zkClient.makePath(arglist.get(0).toString(), true); zkClient.makePath(arglist.get(0).toString(), true);
} else if (line.getOptionValue(CMD).equals(PUT)) { } else if (line.getOptionValue(CMD).equals(PUT)) {
List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE;
List arglist = line.getArgList(); List arglist = line.getArgList();
if (arglist.size() != 2) { if (arglist.size() != 2) {
System.out.println("-" + PUT + " requires two args - the path to create and the data string"); System.out.println("-" + PUT + " requires two args - the path to create and the data string");
System.exit(1); System.exit(1);
} }
zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes(StandardCharsets.UTF_8), zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, true);
acl, CreateMode.PERSISTENT, true);
} else if (line.getOptionValue(CMD).equals(PUT_FILE)) { } else if (line.getOptionValue(CMD).equals(PUT_FILE)) {
List arglist = line.getArgList(); List arglist = line.getArgList();
if (arglist.size() != 2) { if (arglist.size() != 2) {
@ -265,8 +263,7 @@ public class ZkCLI {
} }
InputStream is = new FileInputStream(arglist.get(1).toString()); InputStream is = new FileInputStream(arglist.get(1).toString());
try { try {
zkClient.create(arglist.get(0).toString(), IOUtils.toByteArray(is), zkClient.create(arglist.get(0).toString(), IOUtils.toByteArray(is), CreateMode.PERSISTENT, true);
ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true);
} finally { } finally {
IOUtils.closeQuietly(is); IOUtils.closeQuietly(is);
} }

37
solr/core/src/java/org/apache/solr/cloud/ZkController.java
View File

@ -49,13 +49,17 @@ import org.apache.solr.common.cloud.BeforeReconnect;
import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ClusterStateUtil; import org.apache.solr.common.cloud.ClusterStateUtil;
import org.apache.solr.common.cloud.DefaultConnectionStrategy; import org.apache.solr.common.cloud.DefaultConnectionStrategy;
import org.apache.solr.common.cloud.DefaultZkACLProvider;
import org.apache.solr.common.cloud.DefaultZkCredentialsProvider;
import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.OnReconnect; import org.apache.solr.common.cloud.OnReconnect;
import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkACLProvider;
import org.apache.solr.common.cloud.ZkCmdExecutor; import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkCredentialsProvider;
import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.solr.common.cloud.ZooKeeperException;
@ -212,8 +216,24 @@ public final class ZkController {
this.leaderConflictResolveWait = leaderConflictResolveWait; this.leaderConflictResolveWait = leaderConflictResolveWait;
this.clientTimeout = zkClientTimeout; this.clientTimeout = zkClientTimeout;
DefaultConnectionStrategy strat = new DefaultConnectionStrategy();
String zkACLProviderClass = cc.getConfig().getZkACLProviderClass();
ZkACLProvider zkACLProvider = null;
if (zkACLProviderClass != null && zkACLProviderClass.trim().length() > 0) {
zkACLProvider = cc.getResourceLoader().newInstance(zkACLProviderClass, ZkACLProvider.class);
} else {
zkACLProvider = new DefaultZkACLProvider();
}
String zkCredentialProviderClass = cc.getConfig().getZkCredentialProviderClass();
if (zkCredentialProviderClass != null && zkCredentialProviderClass.trim().length() > 0) {
strat.setZkCredentialsToAddAutomatically(cc.getResourceLoader().newInstance(zkCredentialProviderClass, ZkCredentialsProvider.class));
} else {
strat.setZkCredentialsToAddAutomatically(new DefaultZkCredentialsProvider());
}
zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout, zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout,
zkClientConnectTimeout, new DefaultConnectionStrategy(), zkClientConnectTimeout, strat,
// on reconnect, reload cloud info // on reconnect, reload cloud info
new OnReconnect() { new OnReconnect() {
@ -298,7 +318,7 @@ public final class ZkController {
} }
markAllAsNotLeader(registerOnReconnect); markAllAsNotLeader(registerOnReconnect);
} }
}); }, zkACLProvider);
this.overseerJobQueue = Overseer.getInQueue(zkClient); this.overseerJobQueue = Overseer.getInQueue(zkClient);
this.overseerCollectionQueue = Overseer.getCollectionQueue(zkClient); this.overseerCollectionQueue = Overseer.getCollectionQueue(zkClient);
@ -676,13 +696,14 @@ public final class ZkController {
*/ */
public static boolean checkChrootPath(String zkHost, boolean create) public static boolean checkChrootPath(String zkHost, boolean create)
throws KeeperException, InterruptedException { throws KeeperException, InterruptedException {
if (!containsChroot(zkHost)) { if (!SolrZkClient.containsChroot(zkHost)) {
return true; return true;
} }
log.info("zkHost includes chroot"); log.info("zkHost includes chroot");
String chrootPath = zkHost.substring(zkHost.indexOf("/"), zkHost.length()); String chrootPath = zkHost.substring(zkHost.indexOf("/"), zkHost.length());
SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0, SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0,
zkHost.indexOf("/")), 60 * 1000); zkHost.indexOf("/")), 60000, 30000, null, null, null);
boolean exists = tmpClient.exists(chrootPath, true); boolean exists = tmpClient.exists(chrootPath, true);
if (!exists && create) { if (!exists && create) {
tmpClient.makePath(chrootPath, false, true); tmpClient.makePath(chrootPath, false, true);
@ -692,14 +713,6 @@ public final class ZkController {
return exists; return exists;
} }
/**
* Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions
*/
private static boolean containsChroot(String zkHost) {
return zkHost.contains("/");
}
public boolean isConnected() { public boolean isConnected() {
return zkClient.isConnected(); return zkClient.isConnected();
} }

11
solr/core/src/java/org/apache/solr/core/ConfigSolr.java
View File

@ -215,6 +215,14 @@ public abstract class ConfigSolr {
return get(CfgProp.SOLR_ADMINHANDLER, "org.apache.solr.handler.admin.CoreAdminHandler"); return get(CfgProp.SOLR_ADMINHANDLER, "org.apache.solr.handler.admin.CoreAdminHandler");
} }
public String getZkCredentialProviderClass() {
return get(CfgProp.SOLR_ZKCREDENTIALPROVIDER, null);
}
public String getZkACLProviderClass() {
return get(CfgProp.SOLR_ZKACLPROVIDER, null);
}
public String getCollectionsHandlerClass() { public String getCollectionsHandlerClass() {
return get(CfgProp.SOLR_COLLECTIONSHANDLER, "org.apache.solr.handler.admin.CollectionsHandler"); return get(CfgProp.SOLR_COLLECTIONSHANDLER, "org.apache.solr.handler.admin.CollectionsHandler");
} }
@ -291,6 +299,9 @@ public abstract class ConfigSolr {
SOLR_AUTOREPLICAFAILOVERWORKLOOPDELAY, SOLR_AUTOREPLICAFAILOVERWORKLOOPDELAY,
SOLR_AUTOREPLICAFAILOVERBADNODEEXPIRATION, SOLR_AUTOREPLICAFAILOVERBADNODEEXPIRATION,
SOLR_ZKCREDENTIALPROVIDER,
SOLR_ZKACLPROVIDER,
//TODO: Remove all of these elements for 5.0 //TODO: Remove all of these elements for 5.0
SOLR_PERSISTENT, SOLR_PERSISTENT,
SOLR_CORES_DEFAULT_CORE_NAME, SOLR_CORES_DEFAULT_CORE_NAME,

3
solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
View File

@ -164,6 +164,9 @@ public class ConfigSolrXml extends ConfigSolr {
storeConfigPropertyAsBoolean(s, nl, CfgProp.SOLR_GENERICCORENODENAMES, "genericCoreNodeNames"); storeConfigPropertyAsBoolean(s, nl, CfgProp.SOLR_GENERICCORENODENAMES, "genericCoreNodeNames");
storeConfigPropertyAsString(s, nl, CfgProp.SOLR_ZKACLPROVIDER, "zkACLProvider");
storeConfigPropertyAsString(s, nl, CfgProp.SOLR_ZKCREDENTIALPROVIDER, "zkCredentialProvider");
errorOnLeftOvers(s, nl); errorOnLeftOvers(s, nl);
} }

2
solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
View File

@ -145,6 +145,8 @@ public class ConfigSolrXmlOld extends ConfigSolr {
storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERBADNODEEXPIRATION, "solr/cores/@autoReplicaFailoverBadNodeExpiration"); storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERBADNODEEXPIRATION, "solr/cores/@autoReplicaFailoverBadNodeExpiration");
storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERWAITAFTEREXPIRATION, "solr/cores/@autoReplicaFailoverWaitAfterExpiration"); storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERWAITAFTEREXPIRATION, "solr/cores/@autoReplicaFailoverWaitAfterExpiration");
storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERWORKLOOPDELAY, "solr/cores/@autoReplicaFailoverWorkLoopDelay"); storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERWORKLOOPDELAY, "solr/cores/@autoReplicaFailoverWorkLoopDelay");
storeConfigPropertyAsString(CfgProp.SOLR_ZKACLPROVIDER, "solr/cores/@zkACLProvider");
storeConfigPropertyAsString(CfgProp.SOLR_ZKCREDENTIALPROVIDER, "solr/cores/@zkCredentialProvider");
storeConfigPropertyAsString(CfgProp.SOLR_MANAGEMENTPATH, "solr/cores/@managementPath"); storeConfigPropertyAsString(CfgProp.SOLR_MANAGEMENTPATH, "solr/cores/@managementPath");
storeConfigPropertyAsBoolean(CfgProp.SOLR_SHARESCHEMA, "solr/cores/@shareSchema"); storeConfigPropertyAsBoolean(CfgProp.SOLR_SHARESCHEMA, "solr/cores/@shareSchema");
storeConfigPropertyAsInt(CfgProp.SOLR_TRANSIENTCACHESIZE, "solr/cores/@transientCacheSize"); storeConfigPropertyAsInt(CfgProp.SOLR_TRANSIENTCACHESIZE, "solr/cores/@transientCacheSize");

13
solr/core/src/java/org/apache/solr/core/ZkContainer.java
View File

@ -47,6 +47,9 @@ public class ZkContainer {
private ExecutorService coreZkRegister = Executors.newFixedThreadPool(Integer.MAX_VALUE, private ExecutorService coreZkRegister = Executors.newFixedThreadPool(Integer.MAX_VALUE,
new DefaultSolrThreadFactory("coreZkRegister") ); new DefaultSolrThreadFactory("coreZkRegister") );
// see ZkController.zkRunOnly
private boolean zkRunOnly = Boolean.getBoolean("zkRunOnly"); // expert
public ZkContainer() { public ZkContainer() {
} }
@ -98,7 +101,7 @@ public class ZkContainer {
if (zkRun != null) { if (zkRun != null) {
String zkDataHome = System.getProperty("zkServerDataDir", solrHome + "zoo_data"); String zkDataHome = System.getProperty("zkServerDataDir", solrHome + "zoo_data");
String zkConfHome = System.getProperty("zkServerConfDir", solrHome); String zkConfHome = System.getProperty("zkServerConfDir", solrHome);
zkServer = new SolrZkServer(zkRun, zookeeperHost, zkDataHome, zkConfHome, hostPort); zkServer = new SolrZkServer(stripChroot(zkRun), stripChroot(zookeeperHost), zkDataHome, zkConfHome, hostPort);
zkServer.parseConfig(); zkServer.parseConfig();
zkServer.start(); zkServer.start();
@ -124,9 +127,9 @@ public class ZkContainer {
String confDir = System.getProperty("bootstrap_confdir"); String confDir = System.getProperty("bootstrap_confdir");
boolean boostrapConf = Boolean.getBoolean("bootstrap_conf"); boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
if(!ZkController.checkChrootPath(zookeeperHost, (confDir!=null) || boostrapConf)) { if(!ZkController.checkChrootPath(zookeeperHost, (confDir!=null) || boostrapConf || zkRunOnly)) {
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"A chroot was specified in ZkHost but the znode doesn't exist. "); "A chroot was specified in ZkHost but the znode doesn't exist. " + zookeeperHost);
} }
zkController = new ZkController(cc, zookeeperHost, zkClientTimeout, zkController = new ZkController(cc, zookeeperHost, zkClientTimeout,
zkClientConnectTimeout, host, hostPort, hostContext, zkClientConnectTimeout, host, hostPort, hostContext,
@ -192,6 +195,10 @@ public class ZkContainer {
this.zkController = zkController; this.zkController = zkController;
} }
private String stripChroot(String zkRun) {
return zkRun.substring(0, zkRun.lastIndexOf('/'));
}
public void registerInZk(final SolrCore core, boolean background) { public void registerInZk(final SolrCore core, boolean background) {
Thread thread = new Thread() { Thread thread = new Thread() {
@Override @Override

133
solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java Normal file
View File

@ -0,0 +1,133 @@
package org.apache.solr.cloud;
import java.io.File;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class OutOfBoxZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
protected static Logger log = LoggerFactory
.getLogger(AbstractZkTestCase.class);
private static final Charset DATA_ENCODING = Charset.forName("UTF-8");
protected ZkTestServer zkServer;
protected String zkDir;
@BeforeClass
public static void beforeClass() {
System.setProperty("solrcloud.skip.autorecovery", "true");
}
@AfterClass
public static void afterClass() throws InterruptedException {
System.clearProperty("solrcloud.skip.autorecovery");
}
@Override
public void setUp() throws Exception {
super.setUp();
log.info("####SETUP_START " + getTestName());
createTempDir();
zkDir = createTempDir() + File.separator
+ "zookeeper/server1/data";
log.info("ZooKeeper dataDir:" + zkDir);
zkServer = new ZkTestServer(zkDir);
zkServer.run();
System.setProperty("zkHost", zkServer.getZkAddress());
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT);
zkClient.makePath("/solr", false, true);
zkClient.close();
zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.create("/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.makePath("/unprotectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.close();
log.info("####SETUP_END " + getTestName());
}
@Override
public void tearDown() throws Exception {
zkServer.shutdown();
super.tearDown();
}
@Test
public void testOutOfBoxSolrZkClient() throws Exception {
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true);
} finally {
zkClient.close();
}
}
@Test
public void testOpenACLUnsafeAllover() throws Exception {
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT);
try {
List<String> verifiedList = new ArrayList<String>();
assertOpenACLUnsafeAllover(zkClient, "/", verifiedList);
assertTrue(verifiedList.contains("/solr"));
assertTrue(verifiedList.contains("/solr/unprotectedCreateNode"));
assertTrue(verifiedList.contains("/solr/unprotectedMakePathNode"));
assertTrue(verifiedList.contains("/solr/protectedMakePathNode"));
assertTrue(verifiedList.contains("/solr/protectedCreateNode"));
} finally {
zkClient.close();
}
}
protected void assertOpenACLUnsafeAllover(SolrZkClient zkClient, String path, List<String> verifiedList) throws Exception {
List<ACL> acls = zkClient.getSolrZooKeeper().getACL(path, new Stat());
if (log.isInfoEnabled()) {
log.info("Verifying " + path);
}
assertEquals("Path " + path + " does not have OPEN_ACL_UNSAFE", ZooDefs.Ids.OPEN_ACL_UNSAFE, acls);
verifiedList.add(path);
List<String> children = zkClient.getChildren(path, null, false);
for (String child : children) {
assertOpenACLUnsafeAllover(zkClient, path + ((path.endsWith("/"))?"":"/") + child, verifiedList);
}
}
}

338
solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java Normal file
View File

@ -0,0 +1,338 @@
package org.apache.solr.cloud;
import java.io.File;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.StringUtils;
import org.apache.solr.common.cloud.DefaultZkACLProvider;
import org.apache.solr.common.cloud.DefaultZkCredentialsProvider;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider;
import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider;
import org.apache.solr.common.cloud.ZkACLProvider;
import org.apache.solr.common.cloud.ZkCredentialsProvider;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
protected static Logger log = LoggerFactory
.getLogger(AbstractZkTestCase.class);
private static final Charset DATA_ENCODING = Charset.forName("UTF-8");
protected ZkTestServer zkServer;
protected String zkDir;
@BeforeClass
public static void beforeClass() {
System.setProperty("solrcloud.skip.autorecovery", "true");
}
@AfterClass
public static void afterClass() throws InterruptedException {
System.clearProperty("solrcloud.skip.autorecovery");
}
@Override
public void setUp() throws Exception {
super.setUp();
log.info("####SETUP_START " + getTestName());
createTempDir();
zkDir =createTempDir() + File.separator
+ "zookeeper/server1/data";
log.info("ZooKeeper dataDir:" + zkDir);
zkServer = new ZkTestServer(zkDir);
zkServer.run();
System.setProperty("zkHost", zkServer.getZkAddress());
SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPassword",
"readonlyACLUsername", "readonlyACLPassword").getSolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT);
zkClient.makePath("/solr", false, true);
zkClient.close();
zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPassword",
"readonlyACLUsername", "readonlyACLPassword").getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.close();
zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders(null, null,
null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
zkClient.getSolrZooKeeper().addAuthInfo("digest", ("connectAndAllACLUsername:connectAndAllACLPassword").getBytes(DATA_ENCODING));
zkClient.create("/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.makePath("/unprotectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.close();
log.info("####SETUP_END " + getTestName());
}
@Override
public void tearDown() throws Exception {
zkServer.shutdown();
clearSecuritySystemProperties();
super.tearDown();
}
@Test
public void testNoCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() throws Exception {
SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders(null, null,
null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false);
} finally {
zkClient.close();
}
}
@Test
public void testWrongCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() throws Exception {
SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPasswordWrong",
null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false);
} finally {
zkClient.close();
}
}
@Test
public void testAllCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() throws Exception {
SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPassword",
null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true);
} finally {
zkClient.close();
}
}
@Test
public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() throws Exception {
SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("readonlyACLUsername", "readonlyACLPassword",
null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false);
} finally {
zkClient.close();
}
}
@Test
public void testNoCredentialsSolrZkClientFactoryUsingVMParamsProvidersButWithDifferentVMParamsNames() throws Exception {
useNoCredentials();
SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false);
} finally {
zkClient.close();
}
}
@Test
public void testWrongCredentialsSolrZkClientFactoryUsingVMParamsProvidersButWithDifferentVMParamsNames() throws Exception {
useWrongCredentials();
SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false);
} finally {
zkClient.close();
}
}
@Test
public void testAllCredentialsSolrZkClientFactoryUsingVMParamsProvidersButWithDifferentVMParamsNames() throws Exception {
useAllCredentials();
SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true);
} finally {
zkClient.close();
}
}
@Test
public void testReadonlyCredentialsSolrZkClientFactoryUsingVMParamsProvidersButWithDifferentVMParamsNames() throws Exception {
useReadonlyCredentials();
SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false);
} finally {
zkClient.close();
}
}
private class SolrZkClientFactoryUsingCompletelyNewProviders {
final String digestUsername;
final String digestPassword;
final String digestReadonlyUsername;
final String digestReadonlyPassword;
public SolrZkClientFactoryUsingCompletelyNewProviders(final String digestUsername, final String digestPassword,
final String digestReadonlyUsername, final String digestReadonlyPassword) {
this.digestUsername = digestUsername;
this.digestPassword = digestPassword;
this.digestReadonlyUsername = digestReadonlyUsername;
this.digestReadonlyPassword = digestReadonlyPassword;
}
public SolrZkClient getSolrZkClient(String zkServerAddress, int zkClientTimeout) {
return new SolrZkClient(zkServerAddress, zkClientTimeout) {
@Override
protected ZkCredentialsProvider createZkCredentialsToAddAutomatically() {
return new DefaultZkCredentialsProvider() {
@Override
protected Collection<ZkCredentials> createCredentials() {
List<ZkCredentials> result = new ArrayList<ZkCredentials>();
if (!StringUtils.isEmpty(digestUsername) && !StringUtils.isEmpty(digestPassword)) {
try {
result.add(new ZkCredentials("digest", (digestUsername + ":" + digestPassword).getBytes("UTF-8")));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
return result;
}
};
}
@Override
public ZkACLProvider createZkACLProvider() {
return new DefaultZkACLProvider() {
@Override
protected List<ACL> createGlobalACLsToAdd() {
try {
List<ACL> result = new ArrayList<ACL>();
if (!StringUtils.isEmpty(digestUsername) && !StringUtils.isEmpty(digestPassword)) {
result.add(new ACL(ZooDefs.Perms.ALL, new Id("digest", DigestAuthenticationProvider.generateDigest(digestUsername + ":" + digestPassword))));
}
if (!StringUtils.isEmpty(digestReadonlyUsername) && !StringUtils.isEmpty(digestReadonlyPassword)) {
result.add(new ACL(ZooDefs.Perms.READ, new Id("digest", DigestAuthenticationProvider.generateDigest(digestReadonlyUsername + ":" + digestReadonlyPassword))));
}
if (result.isEmpty()) {
result = ZooDefs.Ids.OPEN_ACL_UNSAFE;
}
return result;
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
};
}
};
}
}
private class SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames extends SolrZkClient {
public SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(String zkServerAddress, int zkClientTimeout) {
super(zkServerAddress, zkClientTimeout);
}
@Override
protected ZkCredentialsProvider createZkCredentialsToAddAutomatically() {
return new VMParamsSingleSetCredentialsDigestZkCredentialsProvider(
"alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME,
"alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME);
}
@Override
public ZkACLProvider createZkACLProvider() {
return new VMParamsAllAndReadonlyDigestZkACLProvider(
"alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME,
"alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME,
"alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME,
"alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME);
}
}
public void useNoCredentials() {
clearSecuritySystemProperties();
}
public void useWrongCredentials() {
clearSecuritySystemProperties();
System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername");
System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPasswordWrong");
}
public void useAllCredentials() {
clearSecuritySystemProperties();
System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername");
System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPassword");
}
public void useReadonlyCredentials() {
clearSecuritySystemProperties();
System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "readonlyACLUsername");
System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "readonlyACLPassword");
}
public void setSecuritySystemProperties() {
System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername");
System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPassword");
System.setProperty("alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, "readonlyACLUsername");
System.setProperty("alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME, "readonlyACLPassword");
}
public void clearSecuritySystemProperties() {
System.clearProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME);
System.clearProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME);
System.clearProperty("alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME);
System.clearProperty("alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME);
}
}

11
solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
View File

@ -318,17 +318,6 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
} }
}).anyTimes(); }).anyTimes();
solrZkClientMock.create(anyObject(String.class), anyObject(byte[].class), anyObject(List.class),anyObject(CreateMode.class), anyBoolean());
expectLastCall().andAnswer(new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
zkMap.put(key, null);
handleCrateCollMessage((byte[]) getCurrentArguments()[1]);
return key;
}
}).anyTimes();
solrZkClientMock.makePath(anyObject(String.class), anyObject(byte[].class), anyBoolean()); solrZkClientMock.makePath(anyObject(String.class), anyObject(byte[].class), anyBoolean());
expectLastCall().andAnswer(new IAnswer<String>() { expectLastCall().andAnswer(new IAnswer<String>() {
@Override @Override

264
solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java Normal file
View File

@ -0,0 +1,264 @@
package org.apache.solr.cloud;
import java.io.File;
import java.nio.charset.Charset;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider;
import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException.NoAuthException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 {
protected static Logger log = LoggerFactory
.getLogger(AbstractZkTestCase.class);
private static final Charset DATA_ENCODING = Charset.forName("UTF-8");
protected ZkTestServer zkServer;
protected String zkDir;
@BeforeClass
public static void beforeClass() {
System.setProperty("solrcloud.skip.autorecovery", "true");
}
@AfterClass
public static void afterClass() throws InterruptedException {
System.clearProperty("solrcloud.skip.autorecovery");
}
@Override
public void setUp() throws Exception {
super.setUp();
log.info("####SETUP_START " + getTestName());
createTempDir();
zkDir = createTempDir() + File.separator
+ "zookeeper/server1/data";
log.info("ZooKeeper dataDir:" + zkDir);
zkServer = new ZkTestServer(zkDir);
zkServer.run();
System.setProperty("zkHost", zkServer.getZkAddress());
setSecuritySystemProperties();
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(),
AbstractZkTestCase.TIMEOUT, 60000, null, null, null);
zkClient.makePath("/solr", false, true);
zkClient.close();
zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.close();
clearSecuritySystemProperties();
zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
// Currently no credentials on ZK connection, because those same VM-params are used for adding ACLs, and here we want
// no (or completely open) ACLs added. Therefore hack your way into being authorized for creating anyway
zkClient.getSolrZooKeeper().addAuthInfo("digest", ("connectAndAllACLUsername:connectAndAllACLPassword").getBytes("UTF-8"));
zkClient.create("/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.makePath("/unprotectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false);
zkClient.close();
log.info("####SETUP_END " + getTestName());
}
@Override
public void tearDown() throws Exception {
zkServer.shutdown();
clearSecuritySystemProperties();
super.tearDown();
}
@Test
public void testNoCredentials() throws Exception {
useNoCredentials();
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
doTest(zkClient, false, false, false, false, false);
} finally {
zkClient.close();
}
}
@Test
public void testWrongCredentials() throws Exception {
useWrongCredentials();
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
doTest(zkClient, false, false, false, false, false);
} finally {
zkClient.close();
}
}
@Test
public void testAllCredentials() throws Exception {
useAllCredentials();
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
doTest(zkClient, true, true, true, true, true);
} finally {
zkClient.close();
}
}
@Test
public void testReadonlyCredentials() throws Exception {
useReadonlyCredentials();
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
try {
doTest(zkClient, true, true, false, false, false);
} finally {
zkClient.close();
}
}
protected static void doTest(SolrZkClient zkClient, boolean getData, boolean list, boolean create, boolean setData, boolean delete) throws Exception {
doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete);
doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete);
doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete);
doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete);
}
protected static void doTest(SolrZkClient zkClient, String path, boolean getData, boolean list, boolean create, boolean setData, boolean delete) throws Exception {
try {
zkClient.getData(path, null, null, false);
if (!getData) fail("NoAuthException expected ");
} catch (NoAuthException nae) {
if (getData) fail("No NoAuthException expected");
// expected
}
try {
zkClient.getChildren(path, null, false);
if (!list) fail("NoAuthException expected ");
} catch (NoAuthException nae) {
if (list) fail("No NoAuthException expected");
// expected
}
try {
zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false);
if (!create) fail("NoAuthException expected ");
else {
zkClient.delete(path + "/subnode", -1, false);
}
} catch (NoAuthException nae) {
if (create) fail("No NoAuthException expected");
// expected
}
try {
zkClient.makePath(path + "/subnode/subsubnode", false);
if (!create) fail("NoAuthException expected ");
else {
zkClient.delete(path + "/subnode/subsubnode", -1, false);
zkClient.delete(path + "/subnode", -1, false);
}
} catch (NoAuthException nae) {
if (create) fail("No NoAuthException expected");
// expected
}
try {
zkClient.setData(path, (byte[])null, false);
if (!setData) fail("NoAuthException expected ");
} catch (NoAuthException nae) {
if (setData) fail("No NoAuthException expected");
// expected
}
try {
// Actually about the ACLs on /solr, but that is protected
zkClient.delete(path, -1, false);
if (!delete) fail("NoAuthException expected ");
} catch (NoAuthException nae) {
if (delete) fail("No NoAuthException expected");
// expected
}
}
private void useNoCredentials() {
clearSecuritySystemProperties();
}
private void useWrongCredentials() {
clearSecuritySystemProperties();
System.setProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName());
System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername");
System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPasswordWrong");
}
private void useAllCredentials() {
clearSecuritySystemProperties();
System.setProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName());
System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername");
System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPassword");
}
private void useReadonlyCredentials() {
clearSecuritySystemProperties();
System.setProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName());
System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "readonlyACLUsername");
System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "readonlyACLPassword");
}
private void setSecuritySystemProperties() {
System.setProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsAllAndReadonlyDigestZkACLProvider.class.getName());
System.setProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName());
System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername");
System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPassword");
System.setProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, "readonlyACLUsername");
System.setProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME, "readonlyACLPassword");
}
private void clearSecuritySystemProperties() {
System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
System.clearProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME);
System.clearProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME);
System.clearProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME);
System.clearProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME);
}
}

5
solr/example/solr/solr.xml
View File

@ -34,6 +34,11 @@
<str name="hostContext">${hostContext:solr}</str> <str name="hostContext">${hostContext:solr}</str>
<int name="zkClientTimeout">${zkClientTimeout:30000}</int> <int name="zkClientTimeout">${zkClientTimeout:30000}</int>
<bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool> <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
<!-- ZooKeeper Security -->
<str name="zkACLProvider">${zkACLProvider:}</str>
<str name="zkCredentialProvider">${zkCredentialProvider:}</str>
</solrcloud> </solrcloud>
<shardHandlerFactory name="shardHandlerFactory" <shardHandlerFactory name="shardHandlerFactory"

26
solr/solrj/src/java/org/apache/solr/common/StringUtils.java Normal file
View File

@ -0,0 +1,26 @@
package org.apache.solr.common;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class StringUtils {
public static boolean isEmpty(String s) {
return (s == null) || s.isEmpty();
}
}

4
solr/solrj/src/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java
View File

@ -34,7 +34,7 @@ public class DefaultConnectionStrategy extends ZkClientConnectionStrategy {
@Override @Override
public void connect(String serverAddress, int timeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException { public void connect(String serverAddress, int timeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException {
SolrZooKeeper zk = new SolrZooKeeper(serverAddress, timeout, watcher); SolrZooKeeper zk = createSolrZooKeeper(serverAddress, timeout, watcher);
boolean success = false; boolean success = false;
try { try {
updater.update(zk); updater.update(zk);
@ -50,7 +50,7 @@ public class DefaultConnectionStrategy extends ZkClientConnectionStrategy {
public void reconnect(final String serverAddress, final int zkClientTimeout, public void reconnect(final String serverAddress, final int zkClientTimeout,
final Watcher watcher, final ZkUpdate updater) throws IOException { final Watcher watcher, final ZkUpdate updater) throws IOException {
log.info("Connection expired - starting a new one..."); log.info("Connection expired - starting a new one...");
SolrZooKeeper zk = new SolrZooKeeper(serverAddress, zkClientTimeout, watcher); SolrZooKeeper zk = createSolrZooKeeper(serverAddress, zkClientTimeout, watcher);
boolean success = false; boolean success = false;
try { try {
updater updater

45
solr/solrj/src/java/org/apache/solr/common/cloud/DefaultZkACLProvider.java Normal file
View File

@ -0,0 +1,45 @@
package org.apache.solr.common.cloud;
import java.util.List;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class DefaultZkACLProvider implements ZkACLProvider {
private List<ACL> globalACLsToAdd;
@Override
public List<ACL> getACLsToAdd(String zNodePath) {
// In default (simple) implementation use the same set of ACLs for all znodes
if (globalACLsToAdd == null) {
synchronized (this) {
if (globalACLsToAdd == null) globalACLsToAdd = createGlobalACLsToAdd();
}
}
return globalACLsToAdd;
}
protected List<ACL> createGlobalACLsToAdd() {
return ZooDefs.Ids.OPEN_ACL_UNSAFE;
}
}

41
solr/solrj/src/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java Normal file
View File

@ -0,0 +1,41 @@
package org.apache.solr.common.cloud;
import java.util.ArrayList;
import java.util.Collection;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class DefaultZkCredentialsProvider implements ZkCredentialsProvider {
private Collection<ZkCredentials> zkCredentials;
@Override
public Collection<ZkCredentials> getCredentials() {
if (zkCredentials == null) {
synchronized (this) {
if (zkCredentials == null) zkCredentials = createCredentials();
}
}
return zkCredentials;
}
protected Collection<ZkCredentials> createCredentials() {
return new ArrayList<ZkCredentials>();
}
}

93
solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
View File

@ -37,6 +37,7 @@ import javax.xml.transform.stream.StreamSource;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.StringUtils;
import org.apache.solr.common.cloud.ZkClientConnectionStrategy.ZkUpdate; import org.apache.solr.common.cloud.ZkClientConnectionStrategy.ZkUpdate;
import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.apache.solr.common.util.SolrjNamedThreadFactory;
@ -47,7 +48,6 @@ import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.KeeperException.NotEmptyException; import org.apache.zookeeper.KeeperException.NotEmptyException;
import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher; import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.data.Stat;
@ -83,6 +83,8 @@ public class SolrZkClient implements Closeable {
private volatile boolean isClosed = false; private volatile boolean isClosed = false;
private ZkClientConnectionStrategy zkClientConnectionStrategy; private ZkClientConnectionStrategy zkClientConnectionStrategy;
private int zkClientTimeout; private int zkClientTimeout;
private ZkACLProvider zkACLProvider;
private String zkServerAddress;
public int getZkClientTimeout() { public int getZkClientTimeout() {
return zkClientTimeout; return zkClientTimeout;
@ -112,17 +114,34 @@ public class SolrZkClient implements Closeable {
public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout, public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout,
ZkClientConnectionStrategy strat, final OnReconnect onReconnect) { ZkClientConnectionStrategy strat, final OnReconnect onReconnect) {
this(zkServerAddress, zkClientTimeout, clientConnectTimeout, strat, onReconnect, null); this(zkServerAddress, zkClientTimeout, clientConnectTimeout, strat, onReconnect, null, null);
} }
public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout, public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout,
ZkClientConnectionStrategy strat, final OnReconnect onReconnect, BeforeReconnect beforeReconnect) { ZkClientConnectionStrategy strat, final OnReconnect onReconnect, BeforeReconnect beforeReconnect) {
this(zkServerAddress, zkClientTimeout, clientConnectTimeout, strat, onReconnect, beforeReconnect, null);
}
public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout,
ZkClientConnectionStrategy strat, final OnReconnect onReconnect, BeforeReconnect beforeReconnect, ZkACLProvider zkACLProvider) {
this.zkClientConnectionStrategy = strat; this.zkClientConnectionStrategy = strat;
this.zkServerAddress = zkServerAddress;
if (strat == null) {
strat = new DefaultConnectionStrategy();
}
if (!strat.hasZkCredentialsToAddAutomatically()) {
ZkCredentialsProvider zkCredentialsToAddAutomatically = createZkCredentialsToAddAutomatically();
strat.setZkCredentialsToAddAutomatically(zkCredentialsToAddAutomatically);
}
this.zkClientTimeout = zkClientTimeout; this.zkClientTimeout = zkClientTimeout;
// we must retry at least as long as the session timeout // we must retry at least as long as the session timeout
zkCmdExecutor = new ZkCmdExecutor(zkClientTimeout); zkCmdExecutor = new ZkCmdExecutor(zkClientTimeout);
connManager = new ConnectionManager("ZooKeeperConnection Watcher:" connManager = new ConnectionManager("ZooKeeperConnection Watcher:"
+ zkServerAddress, this, zkServerAddress, strat, onReconnect, beforeReconnect); + zkServerAddress, this, zkServerAddress, strat, onReconnect, beforeReconnect);
try { try {
strat.connect(zkServerAddress, zkClientTimeout, connManager, strat.connect(zkServerAddress, zkClientTimeout, connManager,
new ZkUpdate() { new ZkUpdate() {
@ -164,6 +183,11 @@ public class SolrZkClient implements Closeable {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} }
numOpens.incrementAndGet(); numOpens.incrementAndGet();
if (zkACLProvider == null) {
this.zkACLProvider = createZkACLProvider();
} else {
this.zkACLProvider = zkACLProvider;
}
} }
public ConnectionManager getConnectionManager() { public ConnectionManager getConnectionManager() {
@ -174,6 +198,38 @@ public class SolrZkClient implements Closeable {
return zkClientConnectionStrategy; return zkClientConnectionStrategy;
} }
public static final String ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME = "zkCredentialsProvider";
protected ZkCredentialsProvider createZkCredentialsToAddAutomatically() {
String zkCredentialsProviderClassName = System.getProperty(ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
if (!StringUtils.isEmpty(zkCredentialsProviderClassName)) {
try {
log.info("Using ZkCredentialsProvider: " + zkCredentialsProviderClassName);
return (ZkCredentialsProvider)Class.forName(zkCredentialsProviderClassName).getConstructor().newInstance();
} catch (Throwable t) {
// just ignore - go default
log.warn("VM param zkCredentialsProvider does not point to a class implementing ZkCredentialsProvider and with a non-arg constructor", t);
}
}
log.info("Using default ZkCredentialsProvider");
return new DefaultZkCredentialsProvider();
}
public static final String ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME = "zkACLProvider";
protected ZkACLProvider createZkACLProvider() {
String zkACLProviderClassName = System.getProperty(ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
if (!StringUtils.isEmpty(zkACLProviderClassName)) {
try {
log.info("Using ZkACLProvider: " + zkACLProviderClassName);
return (ZkACLProvider)Class.forName(zkACLProviderClassName).getConstructor().newInstance();
} catch (Throwable t) {
// just ignore - go default
log.warn("VM param zkACLProvider does not point to a class implementing ZkACLProvider and with a non-arg constructor", t);
}
}
log.info("Using default ZkACLProvider");
return new DefaultZkACLProvider();
}
/** /**
* Returns true if client is connected * Returns true if client is connected
*/ */
@ -262,23 +318,6 @@ public class SolrZkClient implements Closeable {
} }
} }
/**
* Returns path of created node
*/
public String create(final String path, final byte data[], final List<ACL> acl,
final CreateMode createMode, boolean retryOnConnLoss) throws KeeperException, InterruptedException {
if (retryOnConnLoss) {
return zkCmdExecutor.retryOperation(new ZkOperation() {
@Override
public String execute() throws KeeperException, InterruptedException {
return keeper.create(path, data, acl, createMode);
}
});
} else {
return keeper.create(path, data, acl, createMode);
}
}
/** /**
* Returns children of the node at the path * Returns children of the node at the path
*/ */
@ -340,12 +379,13 @@ public class SolrZkClient implements Closeable {
return zkCmdExecutor.retryOperation(new ZkOperation() { return zkCmdExecutor.retryOperation(new ZkOperation() {
@Override @Override
public String execute() throws KeeperException, InterruptedException { public String execute() throws KeeperException, InterruptedException {
return keeper.create(path, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, return keeper.create(path, data, zkACLProvider.getACLsToAdd(path),
createMode); createMode);
} }
}); });
} else { } else {
return keeper.create(path, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, createMode); List<ACL> acls = zkACLProvider.getACLsToAdd(path);
return keeper.create(path, data, acls, createMode);
} }
} }
@ -460,12 +500,12 @@ public class SolrZkClient implements Closeable {
zkCmdExecutor.retryOperation(new ZkOperation() { zkCmdExecutor.retryOperation(new ZkOperation() {
@Override @Override
public Object execute() throws KeeperException, InterruptedException { public Object execute() throws KeeperException, InterruptedException {
keeper.create(currentPath, finalBytes, ZooDefs.Ids.OPEN_ACL_UNSAFE, finalMode); keeper.create(currentPath, finalBytes, zkACLProvider.getACLsToAdd(currentPath), finalMode);
return null; return null;
} }
}); });
} else { } else {
keeper.create(currentPath, bytes, ZooDefs.Ids.OPEN_ACL_UNSAFE, mode); keeper.create(currentPath, bytes, zkACLProvider.getACLsToAdd(currentPath), mode);
} }
} catch (NodeExistsException e) { } catch (NodeExistsException e) {
@ -679,4 +719,11 @@ public class SolrZkClient implements Closeable {
} }
} }
/**
* Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions
*/
public static boolean containsChroot(String zkHost) {
return zkHost.contains("/");
}
} }

89
solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java Normal file
View File

@ -0,0 +1,89 @@
package org.apache.solr.common.cloud;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import org.apache.solr.common.StringUtils;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class VMParamsAllAndReadonlyDigestZkACLProvider extends DefaultZkACLProvider {
public static final String DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME = "zkDigestReadonlyUsername";
public static final String DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME = "zkDigestReadonlyPassword";
final String zkDigestAllUsernameVMParamName;
final String zkDigestAllPasswordVMParamName;
final String zkDigestReadonlyUsernameVMParamName;
final String zkDigestReadonlyPasswordVMParamName;
public VMParamsAllAndReadonlyDigestZkACLProvider() {
this(
VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME,
VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME,
DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME,
DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME
);
}
public VMParamsAllAndReadonlyDigestZkACLProvider(String zkDigestAllUsernameVMParamName, String zkDigestAllPasswordVMParamName,
String zkDigestReadonlyUsernameVMParamName, String zkDigestReadonlyPasswordVMParamName) {
this.zkDigestAllUsernameVMParamName = zkDigestAllUsernameVMParamName;
this.zkDigestAllPasswordVMParamName = zkDigestAllPasswordVMParamName;
this.zkDigestReadonlyUsernameVMParamName = zkDigestReadonlyUsernameVMParamName;
this.zkDigestReadonlyPasswordVMParamName = zkDigestReadonlyPasswordVMParamName;
}
@Override
protected List<ACL> createGlobalACLsToAdd() {
try {
List<ACL> result = new ArrayList<ACL>();
// Not to have to provide too much credentials and ACL information to the process it is assumed that you want "ALL"-acls
// added to the user you are using to connect to ZK (if you are using VMParamsSingleSetCredentialsDigestZkCredentialsProvider)
String digestAllUsername = System.getProperty(zkDigestAllUsernameVMParamName);
String digestAllPassword = System.getProperty(zkDigestAllPasswordVMParamName);
if (!StringUtils.isEmpty(digestAllUsername) && !StringUtils.isEmpty(digestAllPassword)) {
result.add(new ACL(ZooDefs.Perms.ALL, new Id("digest", DigestAuthenticationProvider.generateDigest(digestAllUsername + ":" + digestAllPassword))));
}
// Besides that support for adding additional "READONLY"-acls for another user
String digestReadonlyUsername = System.getProperty(zkDigestReadonlyUsernameVMParamName);
String digestReadonlyPassword = System.getProperty(zkDigestReadonlyPasswordVMParamName);
if (!StringUtils.isEmpty(digestReadonlyUsername) && !StringUtils.isEmpty(digestReadonlyPassword)) {
result.add(new ACL(ZooDefs.Perms.READ, new Id("digest", DigestAuthenticationProvider.generateDigest(digestReadonlyUsername + ":" + digestReadonlyPassword))));
}
if (result.isEmpty()) {
result = super.createGlobalACLsToAdd();
}
return result;
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
}

60
solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java Normal file
View File

@ -0,0 +1,60 @@
package org.apache.solr.common.cloud;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.solr.common.StringUtils;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class VMParamsSingleSetCredentialsDigestZkCredentialsProvider extends DefaultZkCredentialsProvider {
public static final String DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME = "zkDigestUsername";
public static final String DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME = "zkDigestPassword";
final String zkDigestUsernameVMParamName;
final String zkDigestPasswordVMParamName;
public VMParamsSingleSetCredentialsDigestZkCredentialsProvider() {
this(DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME);
}
public VMParamsSingleSetCredentialsDigestZkCredentialsProvider(String zkDigestUsernameVMParamName, String zkDigestPasswordVMParamName) {
this.zkDigestUsernameVMParamName = zkDigestUsernameVMParamName;
this.zkDigestPasswordVMParamName = zkDigestPasswordVMParamName;
}
@Override
protected Collection<ZkCredentials> createCredentials() {
List<ZkCredentials> result = new ArrayList<ZkCredentials>();
String digestUsername = System.getProperty(zkDigestUsernameVMParamName);
String digestPassword = System.getProperty(zkDigestPasswordVMParamName);
if (!StringUtils.isEmpty(digestUsername) && !StringUtils.isEmpty(digestPassword)) {
try {
result.add(new ZkCredentials("digest", (digestUsername + ":" + digestPassword).getBytes("UTF-8")));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
return result;
}
}

28
solr/solrj/src/java/org/apache/solr/common/cloud/ZkACLProvider.java Normal file
View File

@ -0,0 +1,28 @@
package org.apache.solr.common.cloud;
import java.util.List;
import org.apache.zookeeper.data.ACL;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public interface ZkACLProvider {
List<ACL> getACLsToAdd(String zNodePath);
}

30
solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java
View File

@ -23,6 +23,7 @@ import java.util.List;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkCredentialsProvider.ZkCredentials;
import org.apache.zookeeper.Watcher; import org.apache.zookeeper.Watcher;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -33,12 +34,19 @@ import org.slf4j.LoggerFactory;
public abstract class ZkClientConnectionStrategy { public abstract class ZkClientConnectionStrategy {
private static Logger log = LoggerFactory.getLogger(ZkClientConnectionStrategy.class); private static Logger log = LoggerFactory.getLogger(ZkClientConnectionStrategy.class);
private volatile ZkCredentialsProvider zkCredentialsToAddAutomatically;
private volatile boolean zkCredentialsToAddAutomaticallyUsed;
private List<DisconnectedListener> disconnectedListeners = new ArrayList<>(); private List<DisconnectedListener> disconnectedListeners = new ArrayList<>();
private List<ConnectedListener> connectedListeners = new ArrayList<>(); private List<ConnectedListener> connectedListeners = new ArrayList<>();
public abstract void connect(String zkServerAddress, int zkClientTimeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException; public abstract void connect(String zkServerAddress, int zkClientTimeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException;
public abstract void reconnect(String serverAddress, int zkClientTimeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException; public abstract void reconnect(String serverAddress, int zkClientTimeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException;
public ZkClientConnectionStrategy() {
zkCredentialsToAddAutomaticallyUsed = false;
}
public synchronized void disconnected() { public synchronized void disconnected() {
for (DisconnectedListener listener : disconnectedListeners) { for (DisconnectedListener listener : disconnectedListeners) {
try { try {
@ -80,4 +88,26 @@ public abstract class ZkClientConnectionStrategy {
public abstract void update(SolrZooKeeper zooKeeper) throws InterruptedException, TimeoutException, IOException; public abstract void update(SolrZooKeeper zooKeeper) throws InterruptedException, TimeoutException, IOException;
} }
public void setZkCredentialsToAddAutomatically(ZkCredentialsProvider zkCredentialsToAddAutomatically) {
if (zkCredentialsToAddAutomaticallyUsed || (zkCredentialsToAddAutomatically == null))
throw new RuntimeException("Cannot change zkCredentialsToAddAutomatically after it has been (connect or reconnect was called) used or to null");
this.zkCredentialsToAddAutomatically = zkCredentialsToAddAutomatically;
}
public boolean hasZkCredentialsToAddAutomatically() {
return zkCredentialsToAddAutomatically != null;
}
protected SolrZooKeeper createSolrZooKeeper(final String serverAddress, final int zkClientTimeout,
final Watcher watcher) throws IOException {
SolrZooKeeper result = new SolrZooKeeper(serverAddress, zkClientTimeout, watcher);
zkCredentialsToAddAutomaticallyUsed = true;
for (ZkCredentials zkCredentials : zkCredentialsToAddAutomatically.getCredentials()) {
result.addAuthInfo(zkCredentials.getScheme(), zkCredentials.getAuth());
}
return result;
}
} }

13
solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
View File

@ -17,19 +17,14 @@ package org.apache.solr.common.cloud;
* limitations under the License. * limitations under the License.
*/ */
import java.util.List;
import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
public class ZkCmdExecutor { public class ZkCmdExecutor {
private long retryDelay = 1500L; // 1 second would match timeout, so 500 ms over for padding private long retryDelay = 1500L; // 1 second would match timeout, so 500 ms over for padding
private int retryCount; private int retryCount;
private List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE;
private double timeouts; private double timeouts;
/** /**
@ -45,14 +40,6 @@ public class ZkCmdExecutor {
this.retryCount = Math.round(0.5f * ((float)Math.sqrt(8.0f * timeouts + 1.0f) - 1.0f)) + 1; this.retryCount = Math.round(0.5f * ((float)Math.sqrt(8.0f * timeouts + 1.0f) - 1.0f)) + 1;
} }
public List<ACL> getAcl() {
return acl;
}
public void setAcl(List<ACL> acl) {
this.acl = acl;
}
public long getRetryDelay() { public long getRetryDelay() {
return retryDelay; return retryDelay;
} }

45
solr/solrj/src/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java Normal file
View File

@ -0,0 +1,45 @@
package org.apache.solr.common.cloud;
import java.util.Collection;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public interface ZkCredentialsProvider {
public class ZkCredentials {
String scheme;
byte[] auth;
public ZkCredentials(String scheme, byte[] auth) {
super();
this.scheme = scheme;
this.auth = auth;
}
String getScheme() {
return scheme;
}
byte[] getAuth() {
return auth;
}
}
Collection<ZkCredentials> getCredentials();
}

6
solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
View File

@ -29,13 +29,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.ZooDefs;
import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.servlet.ServletHolder;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.common.io.Files;
public class MiniSolrCloudCluster { public class MiniSolrCloudCluster {
private static Logger log = LoggerFactory.getLogger(MiniSolrCloudCluster.class); private static Logger log = LoggerFactory.getLogger(MiniSolrCloudCluster.class);
@ -70,8 +67,7 @@ public class MiniSolrCloudCluster {
AbstractZkTestCase.TIMEOUT, 45000, null); AbstractZkTestCase.TIMEOUT, 45000, null);
zkClient.makePath("/solr", false, true); zkClient.makePath("/solr", false, true);
is = new FileInputStream(solrXml); is = new FileInputStream(solrXml);
zkClient.create("/solr/solr.xml", IOUtils.toByteArray(is), zkClient.create("/solr/solr.xml", IOUtils.toByteArray(is), CreateMode.PERSISTENT, true);
ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true);
} finally { } finally {
IOUtils.closeQuietly(is); IOUtils.closeQuietly(is);
if (zkClient != null) zkClient.close(); if (zkClient != null) zkClient.close();