From a61a530190349049b755d01184de55180c83647a Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Fri, 29 Aug 2014 12:30:53 +0000 Subject: [PATCH] SOLR-4580: Support for protecting content in ZooKeeper. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1621294 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 2 + solr/cloud-dev/clean.sh | 20 ++ solr/cloud-dev/functions.sh | 11 +- solr/cloud-dev/solrcloud-start-existing.sh | 6 +- solr/cloud-dev/solrcloud-start.sh | 10 +- .../org/apache/solr/cloud/DistributedMap.java | 8 +- .../apache/solr/cloud/DistributedQueue.java | 34 +- .../java/org/apache/solr/cloud/Overseer.java | 6 +- .../src/java/org/apache/solr/cloud/ZkCLI.java | 7 +- .../org/apache/solr/cloud/ZkController.java | 37 +- .../java/org/apache/solr/core/ConfigSolr.java | 11 + .../org/apache/solr/core/ConfigSolrXml.java | 3 + .../apache/solr/core/ConfigSolrXmlOld.java | 2 + .../org/apache/solr/core/ZkContainer.java | 13 +- ...OfBoxZkACLAndCredentialsProvidersTest.java | 133 +++++++ ...iddenZkACLAndCredentialsProvidersTest.java | 338 ++++++++++++++++++ .../OverseerCollectionProcessorTest.java | 11 - ...aramsZkACLAndCredentialsProvidersTest.java | 264 ++++++++++++++ solr/example/solr/solr.xml | 5 + .../org/apache/solr/common/StringUtils.java | 26 ++ .../cloud/DefaultConnectionStrategy.java | 4 +- .../common/cloud/DefaultZkACLProvider.java | 45 +++ .../cloud/DefaultZkCredentialsProvider.java | 41 +++ .../solr/common/cloud/SolrZkClient.java | 95 +++-- ...ramsAllAndReadonlyDigestZkACLProvider.java | 89 +++++ ...redentialsDigestZkCredentialsProvider.java | 60 ++++ .../solr/common/cloud/ZkACLProvider.java | 28 ++ .../cloud/ZkClientConnectionStrategy.java | 30 ++ .../solr/common/cloud/ZkCmdExecutor.java | 13 - .../common/cloud/ZkCredentialsProvider.java | 45 +++ .../solr/cloud/MiniSolrCloudCluster.java | 6 +- 31 files changed, 1286 insertions(+), 117 deletions(-) create mode 100755 solr/cloud-dev/clean.sh create mode 100644 solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java create mode 100644 solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java create mode 100644 solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java create mode 100644 solr/solrj/src/java/org/apache/solr/common/StringUtils.java create mode 100644 solr/solrj/src/java/org/apache/solr/common/cloud/DefaultZkACLProvider.java create mode 100644 solr/solrj/src/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java create mode 100644 solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java create mode 100644 solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java create mode 100644 solr/solrj/src/java/org/apache/solr/common/cloud/ZkACLProvider.java create mode 100644 solr/solrj/src/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8132c51dca7..9b99fc305d7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -117,6 +117,8 @@ New Features * SOLR-6403: TransactionLog replay status logging. (Mark Miller) +* SOLR-4580: Support for protecting content in ZooKeeper. (Per Steffensen, Mark Miller) + Bug Fixes ---------------------- diff --git a/solr/cloud-dev/clean.sh b/solr/cloud-dev/clean.sh new file mode 100755 index 00000000000..27f502ef9eb --- /dev/null +++ b/solr/cloud-dev/clean.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +numServers=$1 + +die () { + echo >&2 "$@" + exit 1 +} + +[ "$#" -eq 1 ] || die "1 argument required, $# provided, usage: clean.sh {numServers}" + +cd .. + +for (( i=1; i <= $numServers; i++ )) +do + rm -r -f example$i +done + +rm -r -f examplezk +rm -r -f example-lastlogs \ No newline at end of file diff --git a/solr/cloud-dev/functions.sh b/solr/cloud-dev/functions.sh index a0279659aaf..f83f500d3a6 100755 --- a/solr/cloud-dev/functions.sh +++ b/solr/cloud-dev/functions.sh @@ -2,6 +2,7 @@ INT_JAVA_OPTS="-server -Xms256M -Xmx256M" BASE_PORT=8900 BASE_STOP_PORT=9900 ZK_PORT="2414" +ZK_CHROOT="solr" rebuild() { echo "Rebuilding" @@ -26,17 +27,11 @@ reinstall() { } start() { - OPT="-DzkHost=localhost:$ZK_PORT -DzkRun" + OPT="-DzkHost=localhost:$ZK_PORT/$ZK_CHROOT" NUMSHARDS=$2 echo "Starting instance $1" - if [ "1" = "$1" ]; then - if [ "" = "$NUMSHARDS" ]; then - NUMSHARDS="1" - fi - echo "Instance is running zk, numshards=$NUMSHARDS" - OPT="-DzkRun -Dbootstrap_conf=true -DnumShards=$NUMSHARDS" - fi + setports $1 cd ../example$1 java $JAVA_OPTS -Djetty.port=$PORT $OPT -DSTOP.PORT=$STOP_PORT -DSTOP.KEY=key -jar start.jar 1>example$1.log 2>&1 & diff --git a/solr/cloud-dev/solrcloud-start-existing.sh b/solr/cloud-dev/solrcloud-start-existing.sh index efabbf9e46e..9bdc2986459 100755 --- a/solr/cloud-dev/solrcloud-start-existing.sh +++ b/solr/cloud-dev/solrcloud-start-existing.sh @@ -5,6 +5,8 @@ numServers=$1 baseJettyPort=8900 baseStopPort=9900 +ZK_CHROOT="solr" + die () { echo >&2 "$@" exit 1 @@ -18,7 +20,7 @@ cd .. cd examplezk stopPort=1313 jettyPort=8900 -exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 & +exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkHost=localhost:9900/$ZK_CHROOT -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 & # TODO: we could also remove the default core cd .. @@ -30,5 +32,5 @@ do cd ../example$i stopPort=`expr $baseStopPort + $i` jettyPort=`expr $baseJettyPort + $i` - exec -a jetty java -Xmx1g $JAVA_OPTS -Djetty.port=$jettyPort -DzkHost=localhost:9900 -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 & + exec -a jetty java -Xmx1g $JAVA_OPTS -Djetty.port=$jettyPort -DzkHost=localhost:9900/$ZK_CHROOT -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 & done diff --git a/solr/cloud-dev/solrcloud-start.sh b/solr/cloud-dev/solrcloud-start.sh index b4fb30fb0d4..af30b9302dd 100755 --- a/solr/cloud-dev/solrcloud-start.sh +++ b/solr/cloud-dev/solrcloud-start.sh @@ -3,13 +3,16 @@ # To run on hdfs, try something along the lines of: # export JAVA_OPTS="-Dsolr.directoryFactory=solr.HdfsDirectoryFactory -Dsolr.lock.type=hdfs -Dsolr.hdfs.home=hdfs://localhost:8020/solr -Dsolr.hdfs.confdir=/etc/hadoop_conf/conf" +# To use ZooKeeper security, try: +# export JAVA_OPTS="-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider -DzkDigestUsername=admin-user -DzkDigestPassword=admin-password -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=readonly-password" + numServers=$1 numShards=$2 baseJettyPort=8900 baseStopPort=9900 -zkAddress=localhost:9900 +zkAddress=localhost:9900/solr die () { echo >&2 "$@" @@ -43,6 +46,7 @@ do echo "create example$i" cp -r -f example example$i done + rm -r -f examplezk cp -r -f example examplezk @@ -51,11 +55,11 @@ rm -r -f examplezk/solr/collection1/core.properties cd examplezk stopPort=1313 jettyPort=8900 -exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 & +exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun=localhost:9900/solr -DzkHost=$zkAddress -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 & cd .. # upload config files -java -classpath "example1/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/*" org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost $zkAddress -solrhome example1/solr +java -classpath "example1/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/*" $JAVA_OPTS org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost $zkAddress -solrhome example1/solr cd example diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java index a736a1be579..532de71fd2a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java @@ -47,7 +47,6 @@ public class DistributedMap { private final String dir; private SolrZkClient zookeeper; - private List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; private final String prefix = "mn-"; @@ -66,9 +65,6 @@ public class DistributedMap { throw new SolrException(ErrorCode.SERVER_ERROR, e); } - if (acl != null) { - this.acl = acl; - } this.zookeeper = zookeeper; } @@ -113,10 +109,10 @@ public class DistributedMap { throws KeeperException, InterruptedException { for (;;) { try { - return zookeeper.create(path, data, acl, mode, true); + return zookeeper.create(path, data, mode, true); } catch (KeeperException.NoNodeException e) { try { - zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT, true); + zookeeper.create(dir, new byte[0], CreateMode.PERSISTENT, true); } catch (KeeperException.NodeExistsException ne) { // someone created it } diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java b/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java index d92f5350325..304977b3c23 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java @@ -18,6 +18,14 @@ package org.apache.solr.cloud; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.TreeMap; + import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; @@ -28,19 +36,9 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.TreeMap; - /** * A distributed queue from zk recipes. */ @@ -53,7 +51,6 @@ public class DistributedQueue { private final String dir; private SolrZkClient zookeeper; - private List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; private final String prefix = "qn-"; @@ -61,11 +58,11 @@ public class DistributedQueue { private final Overseer.Stats stats; - public DistributedQueue(SolrZkClient zookeeper, String dir, List acl) { - this(zookeeper, dir, acl, new Overseer.Stats()); + public DistributedQueue(SolrZkClient zookeeper, String dir) { + this(zookeeper, dir, new Overseer.Stats()); } - public DistributedQueue(SolrZkClient zookeeper, String dir, List acl, Overseer.Stats stats) { + public DistributedQueue(SolrZkClient zookeeper, String dir, Overseer.Stats stats) { this.dir = dir; ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zookeeper.getZkClientTimeout()); @@ -78,9 +75,6 @@ public class DistributedQueue { throw new SolrException(ErrorCode.SERVER_ERROR, e); } - if (acl != null) { - this.acl = acl; - } this.zookeeper = zookeeper; this.stats = stats; } @@ -294,7 +288,7 @@ public class DistributedQueue { children = orderedChildren(watcher); break; } catch (KeeperException.NoNodeException e) { - zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT, true); + zookeeper.create(dir, new byte[0], CreateMode.PERSISTENT, true); // go back to the loop and try again } } @@ -366,10 +360,10 @@ public class DistributedQueue { throws KeeperException, InterruptedException { for (;;) { try { - return zookeeper.create(path, data, acl, mode, true); + return zookeeper.create(path, data, mode, true); } catch (KeeperException.NoNodeException e) { try { - zookeeper.create(dir, new byte[0], acl, CreateMode.PERSISTENT, true); + zookeeper.create(dir, new byte[0], CreateMode.PERSISTENT, true); } catch (KeeperException.NodeExistsException ne) { // someone created it } diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java index 15cbc891475..960b5283ce8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java +++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java @@ -1280,13 +1280,13 @@ public class Overseer implements Closeable { static DistributedQueue getInQueue(final SolrZkClient zkClient, Stats zkStats) { createOverseerNode(zkClient); - return new DistributedQueue(zkClient, "/overseer/queue", null, zkStats); + return new DistributedQueue(zkClient, "/overseer/queue", zkStats); } /* Internal queue, not to be used outside of Overseer */ static DistributedQueue getInternalQueue(final SolrZkClient zkClient, Stats zkStats) { createOverseerNode(zkClient); - return new DistributedQueue(zkClient, "/overseer/queue-work", null, zkStats); + return new DistributedQueue(zkClient, "/overseer/queue-work", zkStats); } /* Internal map for failed tasks, not to be used outside of the Overseer */ @@ -1314,7 +1314,7 @@ public class Overseer implements Closeable { static DistributedQueue getCollectionQueue(final SolrZkClient zkClient, Stats zkStats) { createOverseerNode(zkClient); - return new DistributedQueue(zkClient, "/overseer/collection-queue-work", null, zkStats); + return new DistributedQueue(zkClient, "/overseer/collection-queue-work", zkStats); } private static void createOverseerNode(final SolrZkClient zkClient) { diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java index 5c199fe91f7..ae80832d612 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java @@ -249,14 +249,12 @@ public class ZkCLI { } zkClient.makePath(arglist.get(0).toString(), true); } else if (line.getOptionValue(CMD).equals(PUT)) { - List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; List arglist = line.getArgList(); if (arglist.size() != 2) { System.out.println("-" + PUT + " requires two args - the path to create and the data string"); System.exit(1); } - zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes(StandardCharsets.UTF_8), - acl, CreateMode.PERSISTENT, true); + zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, true); } else if (line.getOptionValue(CMD).equals(PUT_FILE)) { List arglist = line.getArgList(); if (arglist.size() != 2) { @@ -265,8 +263,7 @@ public class ZkCLI { } InputStream is = new FileInputStream(arglist.get(1).toString()); try { - zkClient.create(arglist.get(0).toString(), IOUtils.toByteArray(is), - ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true); + zkClient.create(arglist.get(0).toString(), IOUtils.toByteArray(is), CreateMode.PERSISTENT, true); } finally { IOUtils.closeQuietly(is); } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 2df57ff73b7..c5b908db5ae 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -49,13 +49,17 @@ import org.apache.solr.common.cloud.BeforeReconnect; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterStateUtil; import org.apache.solr.common.cloud.DefaultConnectionStrategy; +import org.apache.solr.common.cloud.DefaultZkACLProvider; +import org.apache.solr.common.cloud.DefaultZkCredentialsProvider; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.OnReconnect; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkACLProvider; import org.apache.solr.common.cloud.ZkCmdExecutor; import org.apache.solr.common.cloud.ZkCoreNodeProps; +import org.apache.solr.common.cloud.ZkCredentialsProvider; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZooKeeperException; @@ -212,8 +216,24 @@ public final class ZkController { this.leaderConflictResolveWait = leaderConflictResolveWait; this.clientTimeout = zkClientTimeout; + DefaultConnectionStrategy strat = new DefaultConnectionStrategy(); + String zkACLProviderClass = cc.getConfig().getZkACLProviderClass(); + ZkACLProvider zkACLProvider = null; + if (zkACLProviderClass != null && zkACLProviderClass.trim().length() > 0) { + zkACLProvider = cc.getResourceLoader().newInstance(zkACLProviderClass, ZkACLProvider.class); + } else { + zkACLProvider = new DefaultZkACLProvider(); + } + + String zkCredentialProviderClass = cc.getConfig().getZkCredentialProviderClass(); + if (zkCredentialProviderClass != null && zkCredentialProviderClass.trim().length() > 0) { + strat.setZkCredentialsToAddAutomatically(cc.getResourceLoader().newInstance(zkCredentialProviderClass, ZkCredentialsProvider.class)); + } else { + strat.setZkCredentialsToAddAutomatically(new DefaultZkCredentialsProvider()); + } + zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout, - zkClientConnectTimeout, new DefaultConnectionStrategy(), + zkClientConnectTimeout, strat, // on reconnect, reload cloud info new OnReconnect() { @@ -298,7 +318,7 @@ public final class ZkController { } markAllAsNotLeader(registerOnReconnect); } - }); + }, zkACLProvider); this.overseerJobQueue = Overseer.getInQueue(zkClient); this.overseerCollectionQueue = Overseer.getCollectionQueue(zkClient); @@ -676,13 +696,14 @@ public final class ZkController { */ public static boolean checkChrootPath(String zkHost, boolean create) throws KeeperException, InterruptedException { - if (!containsChroot(zkHost)) { + if (!SolrZkClient.containsChroot(zkHost)) { return true; } log.info("zkHost includes chroot"); String chrootPath = zkHost.substring(zkHost.indexOf("/"), zkHost.length()); + SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0, - zkHost.indexOf("/")), 60 * 1000); + zkHost.indexOf("/")), 60000, 30000, null, null, null); boolean exists = tmpClient.exists(chrootPath, true); if (!exists && create) { tmpClient.makePath(chrootPath, false, true); @@ -692,14 +713,6 @@ public final class ZkController { return exists; } - /** - * Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions - */ - private static boolean containsChroot(String zkHost) { - return zkHost.contains("/"); - } - - public boolean isConnected() { return zkClient.isConnected(); } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSolr.java b/solr/core/src/java/org/apache/solr/core/ConfigSolr.java index fd48580690d..fc1bdf8ee6f 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSolr.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSolr.java @@ -214,7 +214,15 @@ public abstract class ConfigSolr { public String getCoreAdminHandlerClass() { return get(CfgProp.SOLR_ADMINHANDLER, "org.apache.solr.handler.admin.CoreAdminHandler"); } + + public String getZkCredentialProviderClass() { + return get(CfgProp.SOLR_ZKCREDENTIALPROVIDER, null); + } + public String getZkACLProviderClass() { + return get(CfgProp.SOLR_ZKACLPROVIDER, null); + } + public String getCollectionsHandlerClass() { return get(CfgProp.SOLR_COLLECTIONSHANDLER, "org.apache.solr.handler.admin.CollectionsHandler"); } @@ -291,6 +299,9 @@ public abstract class ConfigSolr { SOLR_AUTOREPLICAFAILOVERWORKLOOPDELAY, SOLR_AUTOREPLICAFAILOVERBADNODEEXPIRATION, + SOLR_ZKCREDENTIALPROVIDER, + SOLR_ZKACLPROVIDER, + //TODO: Remove all of these elements for 5.0 SOLR_PERSISTENT, SOLR_CORES_DEFAULT_CORE_NAME, diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java b/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java index 0374155571a..6ded5943b30 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java @@ -164,6 +164,9 @@ public class ConfigSolrXml extends ConfigSolr { storeConfigPropertyAsBoolean(s, nl, CfgProp.SOLR_GENERICCORENODENAMES, "genericCoreNodeNames"); + storeConfigPropertyAsString(s, nl, CfgProp.SOLR_ZKACLPROVIDER, "zkACLProvider"); + storeConfigPropertyAsString(s, nl, CfgProp.SOLR_ZKCREDENTIALPROVIDER, "zkCredentialProvider"); + errorOnLeftOvers(s, nl); } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java b/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java index e2aeac015c5..7bff2759593 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java @@ -145,6 +145,8 @@ public class ConfigSolrXmlOld extends ConfigSolr { storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERBADNODEEXPIRATION, "solr/cores/@autoReplicaFailoverBadNodeExpiration"); storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERWAITAFTEREXPIRATION, "solr/cores/@autoReplicaFailoverWaitAfterExpiration"); storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERWORKLOOPDELAY, "solr/cores/@autoReplicaFailoverWorkLoopDelay"); + storeConfigPropertyAsString(CfgProp.SOLR_ZKACLPROVIDER, "solr/cores/@zkACLProvider"); + storeConfigPropertyAsString(CfgProp.SOLR_ZKCREDENTIALPROVIDER, "solr/cores/@zkCredentialProvider"); storeConfigPropertyAsString(CfgProp.SOLR_MANAGEMENTPATH, "solr/cores/@managementPath"); storeConfigPropertyAsBoolean(CfgProp.SOLR_SHARESCHEMA, "solr/cores/@shareSchema"); storeConfigPropertyAsInt(CfgProp.SOLR_TRANSIENTCACHESIZE, "solr/cores/@transientCacheSize"); diff --git a/solr/core/src/java/org/apache/solr/core/ZkContainer.java b/solr/core/src/java/org/apache/solr/core/ZkContainer.java index a71e3d1775b..c2b5edbf653 100644 --- a/solr/core/src/java/org/apache/solr/core/ZkContainer.java +++ b/solr/core/src/java/org/apache/solr/core/ZkContainer.java @@ -47,6 +47,9 @@ public class ZkContainer { private ExecutorService coreZkRegister = Executors.newFixedThreadPool(Integer.MAX_VALUE, new DefaultSolrThreadFactory("coreZkRegister") ); + // see ZkController.zkRunOnly + private boolean zkRunOnly = Boolean.getBoolean("zkRunOnly"); // expert + public ZkContainer() { } @@ -98,7 +101,7 @@ public class ZkContainer { if (zkRun != null) { String zkDataHome = System.getProperty("zkServerDataDir", solrHome + "zoo_data"); String zkConfHome = System.getProperty("zkServerConfDir", solrHome); - zkServer = new SolrZkServer(zkRun, zookeeperHost, zkDataHome, zkConfHome, hostPort); + zkServer = new SolrZkServer(stripChroot(zkRun), stripChroot(zookeeperHost), zkDataHome, zkConfHome, hostPort); zkServer.parseConfig(); zkServer.start(); @@ -124,9 +127,9 @@ public class ZkContainer { String confDir = System.getProperty("bootstrap_confdir"); boolean boostrapConf = Boolean.getBoolean("bootstrap_conf"); - if(!ZkController.checkChrootPath(zookeeperHost, (confDir!=null) || boostrapConf)) { + if(!ZkController.checkChrootPath(zookeeperHost, (confDir!=null) || boostrapConf || zkRunOnly)) { throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, - "A chroot was specified in ZkHost but the znode doesn't exist. "); + "A chroot was specified in ZkHost but the znode doesn't exist. " + zookeeperHost); } zkController = new ZkController(cc, zookeeperHost, zkClientTimeout, zkClientConnectTimeout, host, hostPort, hostContext, @@ -192,6 +195,10 @@ public class ZkContainer { this.zkController = zkController; } + private String stripChroot(String zkRun) { + return zkRun.substring(0, zkRun.lastIndexOf('/')); + } + public void registerInZk(final SolrCore core, boolean background) { Thread thread = new Thread() { @Override diff --git a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java new file mode 100644 index 00000000000..a59db5fcc79 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java @@ -0,0 +1,133 @@ +package org.apache.solr.cloud; + +import java.io.File; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class OutOfBoxZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { + + protected static Logger log = LoggerFactory + .getLogger(AbstractZkTestCase.class); + + private static final Charset DATA_ENCODING = Charset.forName("UTF-8"); + + protected ZkTestServer zkServer; + + protected String zkDir; + + @BeforeClass + public static void beforeClass() { + System.setProperty("solrcloud.skip.autorecovery", "true"); + } + + @AfterClass + public static void afterClass() throws InterruptedException { + System.clearProperty("solrcloud.skip.autorecovery"); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + log.info("####SETUP_START " + getTestName()); + createTempDir(); + + zkDir = createTempDir() + File.separator + + "zookeeper/server1/data"; + log.info("ZooKeeper dataDir:" + zkDir); + zkServer = new ZkTestServer(zkDir); + zkServer.run(); + + System.setProperty("zkHost", zkServer.getZkAddress()); + + SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT); + zkClient.makePath("/solr", false, true); + zkClient.close(); + + zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.create("/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath("/unprotectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.close(); + + log.info("####SETUP_END " + getTestName()); + } + + @Override + public void tearDown() throws Exception { + zkServer.shutdown(); + + super.tearDown(); + } + + @Test + public void testOutOfBoxSolrZkClient() throws Exception { + SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true); + } finally { + zkClient.close(); + } + } + + @Test + public void testOpenACLUnsafeAllover() throws Exception { + SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT); + try { + List verifiedList = new ArrayList(); + assertOpenACLUnsafeAllover(zkClient, "/", verifiedList); + assertTrue(verifiedList.contains("/solr")); + assertTrue(verifiedList.contains("/solr/unprotectedCreateNode")); + assertTrue(verifiedList.contains("/solr/unprotectedMakePathNode")); + assertTrue(verifiedList.contains("/solr/protectedMakePathNode")); + assertTrue(verifiedList.contains("/solr/protectedCreateNode")); + } finally { + zkClient.close(); + } + } + + + protected void assertOpenACLUnsafeAllover(SolrZkClient zkClient, String path, List verifiedList) throws Exception { + List acls = zkClient.getSolrZooKeeper().getACL(path, new Stat()); + if (log.isInfoEnabled()) { + log.info("Verifying " + path); + } + assertEquals("Path " + path + " does not have OPEN_ACL_UNSAFE", ZooDefs.Ids.OPEN_ACL_UNSAFE, acls); + verifiedList.add(path); + List children = zkClient.getChildren(path, null, false); + for (String child : children) { + assertOpenACLUnsafeAllover(zkClient, path + ((path.endsWith("/"))?"":"/") + child, verifiedList); + } + } + +} diff --git a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java new file mode 100644 index 00000000000..c23987a85d3 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java @@ -0,0 +1,338 @@ +package org.apache.solr.cloud; + +import java.io.File; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.StringUtils; +import org.apache.solr.common.cloud.DefaultZkACLProvider; +import org.apache.solr.common.cloud.DefaultZkCredentialsProvider; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider; +import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider; +import org.apache.solr.common.cloud.ZkACLProvider; +import org.apache.solr.common.cloud.ZkCredentialsProvider; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { + + protected static Logger log = LoggerFactory + .getLogger(AbstractZkTestCase.class); + + private static final Charset DATA_ENCODING = Charset.forName("UTF-8"); + + protected ZkTestServer zkServer; + + protected String zkDir; + + @BeforeClass + public static void beforeClass() { + System.setProperty("solrcloud.skip.autorecovery", "true"); + } + + @AfterClass + public static void afterClass() throws InterruptedException { + System.clearProperty("solrcloud.skip.autorecovery"); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + log.info("####SETUP_START " + getTestName()); + createTempDir(); + + zkDir =createTempDir() + File.separator + + "zookeeper/server1/data"; + log.info("ZooKeeper dataDir:" + zkDir); + zkServer = new ZkTestServer(zkDir); + zkServer.run(); + + System.setProperty("zkHost", zkServer.getZkAddress()); + + SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPassword", + "readonlyACLUsername", "readonlyACLPassword").getSolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT); + zkClient.makePath("/solr", false, true); + zkClient.close(); + + zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPassword", + "readonlyACLUsername", "readonlyACLPassword").getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.close(); + + zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, + null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + zkClient.getSolrZooKeeper().addAuthInfo("digest", ("connectAndAllACLUsername:connectAndAllACLPassword").getBytes(DATA_ENCODING)); + zkClient.create("/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath("/unprotectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.close(); + + log.info("####SETUP_END " + getTestName()); + } + + @Override + public void tearDown() throws Exception { + zkServer.shutdown(); + + clearSecuritySystemProperties(); + + super.tearDown(); + } + + @Test + public void testNoCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() throws Exception { + SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, + null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false); + } finally { + zkClient.close(); + } + } + + @Test + public void testWrongCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() throws Exception { + SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPasswordWrong", + null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false); + } finally { + zkClient.close(); + } + } + + @Test + public void testAllCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() throws Exception { + SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPassword", + null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true); + } finally { + zkClient.close(); + } + } + + @Test + public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() throws Exception { + SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("readonlyACLUsername", "readonlyACLPassword", + null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false); + } finally { + zkClient.close(); + } + } + + @Test + public void testNoCredentialsSolrZkClientFactoryUsingVMParamsProvidersButWithDifferentVMParamsNames() throws Exception { + useNoCredentials(); + + SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false); + } finally { + zkClient.close(); + } + } + + @Test + public void testWrongCredentialsSolrZkClientFactoryUsingVMParamsProvidersButWithDifferentVMParamsNames() throws Exception { + useWrongCredentials(); + + SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false); + } finally { + zkClient.close(); + } + } + + @Test + public void testAllCredentialsSolrZkClientFactoryUsingVMParamsProvidersButWithDifferentVMParamsNames() throws Exception { + useAllCredentials(); + + SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true); + } finally { + zkClient.close(); + } + } + + @Test + public void testReadonlyCredentialsSolrZkClientFactoryUsingVMParamsProvidersButWithDifferentVMParamsNames() throws Exception { + useReadonlyCredentials(); + + SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false); + } finally { + zkClient.close(); + } + } + + private class SolrZkClientFactoryUsingCompletelyNewProviders { + + final String digestUsername; + final String digestPassword; + final String digestReadonlyUsername; + final String digestReadonlyPassword; + + public SolrZkClientFactoryUsingCompletelyNewProviders(final String digestUsername, final String digestPassword, + final String digestReadonlyUsername, final String digestReadonlyPassword) { + this.digestUsername = digestUsername; + this.digestPassword = digestPassword; + this.digestReadonlyUsername = digestReadonlyUsername; + this.digestReadonlyPassword = digestReadonlyPassword; + } + + public SolrZkClient getSolrZkClient(String zkServerAddress, int zkClientTimeout) { + return new SolrZkClient(zkServerAddress, zkClientTimeout) { + + @Override + protected ZkCredentialsProvider createZkCredentialsToAddAutomatically() { + return new DefaultZkCredentialsProvider() { + @Override + protected Collection createCredentials() { + List result = new ArrayList(); + if (!StringUtils.isEmpty(digestUsername) && !StringUtils.isEmpty(digestPassword)) { + try { + result.add(new ZkCredentials("digest", (digestUsername + ":" + digestPassword).getBytes("UTF-8"))); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + return result; + } + + }; + } + + @Override + public ZkACLProvider createZkACLProvider() { + return new DefaultZkACLProvider() { + @Override + protected List createGlobalACLsToAdd() { + try { + List result = new ArrayList(); + + if (!StringUtils.isEmpty(digestUsername) && !StringUtils.isEmpty(digestPassword)) { + result.add(new ACL(ZooDefs.Perms.ALL, new Id("digest", DigestAuthenticationProvider.generateDigest(digestUsername + ":" + digestPassword)))); + } + + if (!StringUtils.isEmpty(digestReadonlyUsername) && !StringUtils.isEmpty(digestReadonlyPassword)) { + result.add(new ACL(ZooDefs.Perms.READ, new Id("digest", DigestAuthenticationProvider.generateDigest(digestReadonlyUsername + ":" + digestReadonlyPassword)))); + } + + if (result.isEmpty()) { + result = ZooDefs.Ids.OPEN_ACL_UNSAFE; + } + + return result; + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + }; + } + + }; + } + + } + + private class SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames extends SolrZkClient { + + public SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(String zkServerAddress, int zkClientTimeout) { + super(zkServerAddress, zkClientTimeout); + } + + @Override + protected ZkCredentialsProvider createZkCredentialsToAddAutomatically() { + return new VMParamsSingleSetCredentialsDigestZkCredentialsProvider( + "alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, + "alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME); + } + + @Override + public ZkACLProvider createZkACLProvider() { + return new VMParamsAllAndReadonlyDigestZkACLProvider( + "alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, + "alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, + "alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, + "alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME); + } + } + + public void useNoCredentials() { + clearSecuritySystemProperties(); + } + + public void useWrongCredentials() { + clearSecuritySystemProperties(); + + System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername"); + System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPasswordWrong"); + } + + public void useAllCredentials() { + clearSecuritySystemProperties(); + + System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername"); + System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPassword"); + } + + public void useReadonlyCredentials() { + clearSecuritySystemProperties(); + + System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "readonlyACLUsername"); + System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "readonlyACLPassword"); + } + + public void setSecuritySystemProperties() { + System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername"); + System.setProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPassword"); + System.setProperty("alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, "readonlyACLUsername"); + System.setProperty("alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME, "readonlyACLPassword"); + } + + public void clearSecuritySystemProperties() { + System.clearProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME); + System.clearProperty("alternative" + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME); + System.clearProperty("alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME); + System.clearProperty("alternative" + VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME); + } + +} + diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java index 7b08dbe9d61..c8a22aee3fa 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java @@ -318,17 +318,6 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 { } }).anyTimes(); - solrZkClientMock.create(anyObject(String.class), anyObject(byte[].class), anyObject(List.class),anyObject(CreateMode.class), anyBoolean()); - expectLastCall().andAnswer(new IAnswer() { - @Override - public String answer() throws Throwable { - String key = (String) getCurrentArguments()[0]; - zkMap.put(key, null); - handleCrateCollMessage((byte[]) getCurrentArguments()[1]); - return key; - } - }).anyTimes(); - solrZkClientMock.makePath(anyObject(String.class), anyObject(byte[].class), anyBoolean()); expectLastCall().andAnswer(new IAnswer() { @Override diff --git a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java new file mode 100644 index 00000000000..e4c66534175 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java @@ -0,0 +1,264 @@ +package org.apache.solr.cloud; + +import java.io.File; +import java.nio.charset.Charset; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider; +import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException.NoAuthException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { + + protected static Logger log = LoggerFactory + .getLogger(AbstractZkTestCase.class); + + private static final Charset DATA_ENCODING = Charset.forName("UTF-8"); + + protected ZkTestServer zkServer; + + protected String zkDir; + + @BeforeClass + public static void beforeClass() { + System.setProperty("solrcloud.skip.autorecovery", "true"); + } + + @AfterClass + public static void afterClass() throws InterruptedException { + System.clearProperty("solrcloud.skip.autorecovery"); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + log.info("####SETUP_START " + getTestName()); + createTempDir(); + + zkDir = createTempDir() + File.separator + + "zookeeper/server1/data"; + log.info("ZooKeeper dataDir:" + zkDir); + zkServer = new ZkTestServer(zkDir); + zkServer.run(); + + System.setProperty("zkHost", zkServer.getZkAddress()); + + setSecuritySystemProperties(); + + SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), + AbstractZkTestCase.TIMEOUT, 60000, null, null, null); + zkClient.makePath("/solr", false, true); + zkClient.close(); + + zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.close(); + + clearSecuritySystemProperties(); + + zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + // Currently no credentials on ZK connection, because those same VM-params are used for adding ACLs, and here we want + // no (or completely open) ACLs added. Therefore hack your way into being authorized for creating anyway + zkClient.getSolrZooKeeper().addAuthInfo("digest", ("connectAndAllACLUsername:connectAndAllACLPassword").getBytes("UTF-8")); + zkClient.create("/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath("/unprotectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.close(); + + log.info("####SETUP_END " + getTestName()); + } + + @Override + public void tearDown() throws Exception { + zkServer.shutdown(); + + clearSecuritySystemProperties(); + + super.tearDown(); + } + + @Test + public void testNoCredentials() throws Exception { + useNoCredentials(); + + SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + doTest(zkClient, false, false, false, false, false); + } finally { + zkClient.close(); + } + } + + @Test + public void testWrongCredentials() throws Exception { + useWrongCredentials(); + + SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + doTest(zkClient, false, false, false, false, false); + } finally { + zkClient.close(); + } + } + + @Test + public void testAllCredentials() throws Exception { + useAllCredentials(); + + SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + doTest(zkClient, true, true, true, true, true); + } finally { + zkClient.close(); + } + } + + @Test + public void testReadonlyCredentials() throws Exception { + useReadonlyCredentials(); + + SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); + try { + doTest(zkClient, true, true, false, false, false); + } finally { + zkClient.close(); + } + } + + protected static void doTest(SolrZkClient zkClient, boolean getData, boolean list, boolean create, boolean setData, boolean delete) throws Exception { + doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete); + doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete); + doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete); + doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete); + } + + protected static void doTest(SolrZkClient zkClient, String path, boolean getData, boolean list, boolean create, boolean setData, boolean delete) throws Exception { + try { + zkClient.getData(path, null, null, false); + if (!getData) fail("NoAuthException expected "); + } catch (NoAuthException nae) { + if (getData) fail("No NoAuthException expected"); + // expected + } + + try { + zkClient.getChildren(path, null, false); + if (!list) fail("NoAuthException expected "); + } catch (NoAuthException nae) { + if (list) fail("No NoAuthException expected"); + // expected + } + + try { + zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false); + if (!create) fail("NoAuthException expected "); + else { + zkClient.delete(path + "/subnode", -1, false); + } + } catch (NoAuthException nae) { + if (create) fail("No NoAuthException expected"); + // expected + } + + try { + zkClient.makePath(path + "/subnode/subsubnode", false); + if (!create) fail("NoAuthException expected "); + else { + zkClient.delete(path + "/subnode/subsubnode", -1, false); + zkClient.delete(path + "/subnode", -1, false); + } + } catch (NoAuthException nae) { + if (create) fail("No NoAuthException expected"); + // expected + } + + try { + zkClient.setData(path, (byte[])null, false); + if (!setData) fail("NoAuthException expected "); + } catch (NoAuthException nae) { + if (setData) fail("No NoAuthException expected"); + // expected + } + + try { + // Actually about the ACLs on /solr, but that is protected + zkClient.delete(path, -1, false); + if (!delete) fail("NoAuthException expected "); + } catch (NoAuthException nae) { + if (delete) fail("No NoAuthException expected"); + // expected + } + + } + + private void useNoCredentials() { + clearSecuritySystemProperties(); + } + + private void useWrongCredentials() { + clearSecuritySystemProperties(); + + System.setProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName()); + System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername"); + System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPasswordWrong"); + } + + private void useAllCredentials() { + clearSecuritySystemProperties(); + + System.setProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName()); + System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername"); + System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPassword"); + } + + private void useReadonlyCredentials() { + clearSecuritySystemProperties(); + + System.setProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName()); + System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "readonlyACLUsername"); + System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "readonlyACLPassword"); + } + + private void setSecuritySystemProperties() { + System.setProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsAllAndReadonlyDigestZkACLProvider.class.getName()); + System.setProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName()); + System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "connectAndAllACLUsername"); + System.setProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "connectAndAllACLPassword"); + System.setProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, "readonlyACLUsername"); + System.setProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME, "readonlyACLPassword"); + } + + private void clearSecuritySystemProperties() { + System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + System.clearProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME); + System.clearProperty(VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME); + System.clearProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME); + System.clearProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME); + } + +} diff --git a/solr/example/solr/solr.xml b/solr/example/solr/solr.xml index 94d60b6a29b..d4bb96b039a 100644 --- a/solr/example/solr/solr.xml +++ b/solr/example/solr/solr.xml @@ -34,6 +34,11 @@ ${hostContext:solr} ${zkClientTimeout:30000} ${genericCoreNodeNames:true} + + + ${zkACLProvider:} + ${zkCredentialProvider:} + globalACLsToAdd; + + @Override + public List getACLsToAdd(String zNodePath) { + // In default (simple) implementation use the same set of ACLs for all znodes + if (globalACLsToAdd == null) { + synchronized (this) { + if (globalACLsToAdd == null) globalACLsToAdd = createGlobalACLsToAdd(); + } + } + return globalACLsToAdd; + + } + + protected List createGlobalACLsToAdd() { + return ZooDefs.Ids.OPEN_ACL_UNSAFE; + } + +} diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java new file mode 100644 index 00000000000..ca09068e6c0 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java @@ -0,0 +1,41 @@ +package org.apache.solr.common.cloud; + +import java.util.ArrayList; +import java.util.Collection; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class DefaultZkCredentialsProvider implements ZkCredentialsProvider { + + private Collection zkCredentials; + + @Override + public Collection getCredentials() { + if (zkCredentials == null) { + synchronized (this) { + if (zkCredentials == null) zkCredentials = createCredentials(); + } + } + return zkCredentials; + } + + protected Collection createCredentials() { + return new ArrayList(); + } + +} diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java index b407ad097d6..ed01091a711 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java @@ -37,6 +37,7 @@ import javax.xml.transform.stream.StreamSource; import org.apache.commons.io.FileUtils; import org.apache.solr.common.SolrException; +import org.apache.solr.common.StringUtils; import org.apache.solr.common.cloud.ZkClientConnectionStrategy.ZkUpdate; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.SolrjNamedThreadFactory; @@ -47,7 +48,6 @@ import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.KeeperException.NotEmptyException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; @@ -83,6 +83,8 @@ public class SolrZkClient implements Closeable { private volatile boolean isClosed = false; private ZkClientConnectionStrategy zkClientConnectionStrategy; private int zkClientTimeout; + private ZkACLProvider zkACLProvider; + private String zkServerAddress; public int getZkClientTimeout() { return zkClientTimeout; @@ -112,17 +114,34 @@ public class SolrZkClient implements Closeable { public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout, ZkClientConnectionStrategy strat, final OnReconnect onReconnect) { - this(zkServerAddress, zkClientTimeout, clientConnectTimeout, strat, onReconnect, null); + this(zkServerAddress, zkClientTimeout, clientConnectTimeout, strat, onReconnect, null, null); + } + + public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout, + ZkClientConnectionStrategy strat, final OnReconnect onReconnect, BeforeReconnect beforeReconnect) { + this(zkServerAddress, zkClientTimeout, clientConnectTimeout, strat, onReconnect, beforeReconnect, null); } public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout, - ZkClientConnectionStrategy strat, final OnReconnect onReconnect, BeforeReconnect beforeReconnect) { + ZkClientConnectionStrategy strat, final OnReconnect onReconnect, BeforeReconnect beforeReconnect, ZkACLProvider zkACLProvider) { this.zkClientConnectionStrategy = strat; + this.zkServerAddress = zkServerAddress; + + if (strat == null) { + strat = new DefaultConnectionStrategy(); + } + + if (!strat.hasZkCredentialsToAddAutomatically()) { + ZkCredentialsProvider zkCredentialsToAddAutomatically = createZkCredentialsToAddAutomatically(); + strat.setZkCredentialsToAddAutomatically(zkCredentialsToAddAutomatically); + } + this.zkClientTimeout = zkClientTimeout; // we must retry at least as long as the session timeout zkCmdExecutor = new ZkCmdExecutor(zkClientTimeout); connManager = new ConnectionManager("ZooKeeperConnection Watcher:" + zkServerAddress, this, zkServerAddress, strat, onReconnect, beforeReconnect); + try { strat.connect(zkServerAddress, zkClientTimeout, connManager, new ZkUpdate() { @@ -164,6 +183,11 @@ public class SolrZkClient implements Closeable { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } numOpens.incrementAndGet(); + if (zkACLProvider == null) { + this.zkACLProvider = createZkACLProvider(); + } else { + this.zkACLProvider = zkACLProvider; + } } public ConnectionManager getConnectionManager() { @@ -174,6 +198,38 @@ public class SolrZkClient implements Closeable { return zkClientConnectionStrategy; } + public static final String ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME = "zkCredentialsProvider"; + protected ZkCredentialsProvider createZkCredentialsToAddAutomatically() { + String zkCredentialsProviderClassName = System.getProperty(ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + if (!StringUtils.isEmpty(zkCredentialsProviderClassName)) { + try { + log.info("Using ZkCredentialsProvider: " + zkCredentialsProviderClassName); + return (ZkCredentialsProvider)Class.forName(zkCredentialsProviderClassName).getConstructor().newInstance(); + } catch (Throwable t) { + // just ignore - go default + log.warn("VM param zkCredentialsProvider does not point to a class implementing ZkCredentialsProvider and with a non-arg constructor", t); + } + } + log.info("Using default ZkCredentialsProvider"); + return new DefaultZkCredentialsProvider(); + } + + public static final String ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME = "zkACLProvider"; + protected ZkACLProvider createZkACLProvider() { + String zkACLProviderClassName = System.getProperty(ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + if (!StringUtils.isEmpty(zkACLProviderClassName)) { + try { + log.info("Using ZkACLProvider: " + zkACLProviderClassName); + return (ZkACLProvider)Class.forName(zkACLProviderClassName).getConstructor().newInstance(); + } catch (Throwable t) { + // just ignore - go default + log.warn("VM param zkACLProvider does not point to a class implementing ZkACLProvider and with a non-arg constructor", t); + } + } + log.info("Using default ZkACLProvider"); + return new DefaultZkACLProvider(); + } + /** * Returns true if client is connected */ @@ -262,23 +318,6 @@ public class SolrZkClient implements Closeable { } } - /** - * Returns path of created node - */ - public String create(final String path, final byte data[], final List acl, - final CreateMode createMode, boolean retryOnConnLoss) throws KeeperException, InterruptedException { - if (retryOnConnLoss) { - return zkCmdExecutor.retryOperation(new ZkOperation() { - @Override - public String execute() throws KeeperException, InterruptedException { - return keeper.create(path, data, acl, createMode); - } - }); - } else { - return keeper.create(path, data, acl, createMode); - } - } - /** * Returns children of the node at the path */ @@ -340,12 +379,13 @@ public class SolrZkClient implements Closeable { return zkCmdExecutor.retryOperation(new ZkOperation() { @Override public String execute() throws KeeperException, InterruptedException { - return keeper.create(path, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, + return keeper.create(path, data, zkACLProvider.getACLsToAdd(path), createMode); } }); } else { - return keeper.create(path, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, createMode); + List acls = zkACLProvider.getACLsToAdd(path); + return keeper.create(path, data, acls, createMode); } } @@ -460,12 +500,12 @@ public class SolrZkClient implements Closeable { zkCmdExecutor.retryOperation(new ZkOperation() { @Override public Object execute() throws KeeperException, InterruptedException { - keeper.create(currentPath, finalBytes, ZooDefs.Ids.OPEN_ACL_UNSAFE, finalMode); + keeper.create(currentPath, finalBytes, zkACLProvider.getACLsToAdd(currentPath), finalMode); return null; } }); } else { - keeper.create(currentPath, bytes, ZooDefs.Ids.OPEN_ACL_UNSAFE, mode); + keeper.create(currentPath, bytes, zkACLProvider.getACLsToAdd(currentPath), mode); } } catch (NodeExistsException e) { @@ -678,5 +718,12 @@ public class SolrZkClient implements Closeable { return; } } + + /** + * Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions + */ + public static boolean containsChroot(String zkHost) { + return zkHost.contains("/"); + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java b/solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java new file mode 100644 index 00000000000..0b9ae1db41b --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java @@ -0,0 +1,89 @@ +package org.apache.solr.common.cloud; + +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.solr.common.StringUtils; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class VMParamsAllAndReadonlyDigestZkACLProvider extends DefaultZkACLProvider { + + public static final String DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME = "zkDigestReadonlyUsername"; + public static final String DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME = "zkDigestReadonlyPassword"; + + final String zkDigestAllUsernameVMParamName; + final String zkDigestAllPasswordVMParamName; + final String zkDigestReadonlyUsernameVMParamName; + final String zkDigestReadonlyPasswordVMParamName; + + public VMParamsAllAndReadonlyDigestZkACLProvider() { + this( + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, + VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, + DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, + DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME + ); + } + + public VMParamsAllAndReadonlyDigestZkACLProvider(String zkDigestAllUsernameVMParamName, String zkDigestAllPasswordVMParamName, + String zkDigestReadonlyUsernameVMParamName, String zkDigestReadonlyPasswordVMParamName) { + this.zkDigestAllUsernameVMParamName = zkDigestAllUsernameVMParamName; + this.zkDigestAllPasswordVMParamName = zkDigestAllPasswordVMParamName; + this.zkDigestReadonlyUsernameVMParamName = zkDigestReadonlyUsernameVMParamName; + this.zkDigestReadonlyPasswordVMParamName = zkDigestReadonlyPasswordVMParamName; + } + + + @Override + protected List createGlobalACLsToAdd() { + try { + List result = new ArrayList(); + + // Not to have to provide too much credentials and ACL information to the process it is assumed that you want "ALL"-acls + // added to the user you are using to connect to ZK (if you are using VMParamsSingleSetCredentialsDigestZkCredentialsProvider) + String digestAllUsername = System.getProperty(zkDigestAllUsernameVMParamName); + String digestAllPassword = System.getProperty(zkDigestAllPasswordVMParamName); + if (!StringUtils.isEmpty(digestAllUsername) && !StringUtils.isEmpty(digestAllPassword)) { + result.add(new ACL(ZooDefs.Perms.ALL, new Id("digest", DigestAuthenticationProvider.generateDigest(digestAllUsername + ":" + digestAllPassword)))); + } + + // Besides that support for adding additional "READONLY"-acls for another user + String digestReadonlyUsername = System.getProperty(zkDigestReadonlyUsernameVMParamName); + String digestReadonlyPassword = System.getProperty(zkDigestReadonlyPasswordVMParamName); + if (!StringUtils.isEmpty(digestReadonlyUsername) && !StringUtils.isEmpty(digestReadonlyPassword)) { + result.add(new ACL(ZooDefs.Perms.READ, new Id("digest", DigestAuthenticationProvider.generateDigest(digestReadonlyUsername + ":" + digestReadonlyPassword)))); + } + + if (result.isEmpty()) { + result = super.createGlobalACLsToAdd(); + } + + return result; + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + +} + diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java b/solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java new file mode 100644 index 00000000000..1e575fdb20d --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java @@ -0,0 +1,60 @@ +package org.apache.solr.common.cloud; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.solr.common.StringUtils; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class VMParamsSingleSetCredentialsDigestZkCredentialsProvider extends DefaultZkCredentialsProvider { + + public static final String DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME = "zkDigestUsername"; + public static final String DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME = "zkDigestPassword"; + + final String zkDigestUsernameVMParamName; + final String zkDigestPasswordVMParamName; + + public VMParamsSingleSetCredentialsDigestZkCredentialsProvider() { + this(DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME); + } + + public VMParamsSingleSetCredentialsDigestZkCredentialsProvider(String zkDigestUsernameVMParamName, String zkDigestPasswordVMParamName) { + this.zkDigestUsernameVMParamName = zkDigestUsernameVMParamName; + this.zkDigestPasswordVMParamName = zkDigestPasswordVMParamName; + } + + @Override + protected Collection createCredentials() { + List result = new ArrayList(); + String digestUsername = System.getProperty(zkDigestUsernameVMParamName); + String digestPassword = System.getProperty(zkDigestPasswordVMParamName); + if (!StringUtils.isEmpty(digestUsername) && !StringUtils.isEmpty(digestPassword)) { + try { + result.add(new ZkCredentials("digest", (digestUsername + ":" + digestPassword).getBytes("UTF-8"))); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + return result; + } + +} + diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkACLProvider.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkACLProvider.java new file mode 100644 index 00000000000..03149b3d4ba --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkACLProvider.java @@ -0,0 +1,28 @@ +package org.apache.solr.common.cloud; + +import java.util.List; + +import org.apache.zookeeper.data.ACL; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public interface ZkACLProvider { + + List getACLsToAdd(String zNodePath); + +} diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java index 5dec65d105e..5f4baa5cbd2 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.concurrent.TimeoutException; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkCredentialsProvider.ZkCredentials; import org.apache.zookeeper.Watcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,12 +34,19 @@ import org.slf4j.LoggerFactory; public abstract class ZkClientConnectionStrategy { private static Logger log = LoggerFactory.getLogger(ZkClientConnectionStrategy.class); + private volatile ZkCredentialsProvider zkCredentialsToAddAutomatically; + private volatile boolean zkCredentialsToAddAutomaticallyUsed; + private List disconnectedListeners = new ArrayList<>(); private List connectedListeners = new ArrayList<>(); public abstract void connect(String zkServerAddress, int zkClientTimeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException; public abstract void reconnect(String serverAddress, int zkClientTimeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException; + public ZkClientConnectionStrategy() { + zkCredentialsToAddAutomaticallyUsed = false; + } + public synchronized void disconnected() { for (DisconnectedListener listener : disconnectedListeners) { try { @@ -80,4 +88,26 @@ public abstract class ZkClientConnectionStrategy { public abstract void update(SolrZooKeeper zooKeeper) throws InterruptedException, TimeoutException, IOException; } + public void setZkCredentialsToAddAutomatically(ZkCredentialsProvider zkCredentialsToAddAutomatically) { + if (zkCredentialsToAddAutomaticallyUsed || (zkCredentialsToAddAutomatically == null)) + throw new RuntimeException("Cannot change zkCredentialsToAddAutomatically after it has been (connect or reconnect was called) used or to null"); + this.zkCredentialsToAddAutomatically = zkCredentialsToAddAutomatically; + } + + public boolean hasZkCredentialsToAddAutomatically() { + return zkCredentialsToAddAutomatically != null; + } + + protected SolrZooKeeper createSolrZooKeeper(final String serverAddress, final int zkClientTimeout, + final Watcher watcher) throws IOException { + SolrZooKeeper result = new SolrZooKeeper(serverAddress, zkClientTimeout, watcher); + + zkCredentialsToAddAutomaticallyUsed = true; + for (ZkCredentials zkCredentials : zkCredentialsToAddAutomatically.getCredentials()) { + result.addAuthInfo(zkCredentials.getScheme(), zkCredentials.getAuth()); + } + + return result; + } + } diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java index bcbb5fb2901..36aa045e009 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java @@ -17,19 +17,14 @@ package org.apache.solr.common.cloud; * limitations under the License. */ -import java.util.List; - import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NodeExistsException; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; public class ZkCmdExecutor { private long retryDelay = 1500L; // 1 second would match timeout, so 500 ms over for padding private int retryCount; - private List acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; private double timeouts; /** @@ -45,14 +40,6 @@ public class ZkCmdExecutor { this.retryCount = Math.round(0.5f * ((float)Math.sqrt(8.0f * timeouts + 1.0f) - 1.0f)) + 1; } - public List getAcl() { - return acl; - } - - public void setAcl(List acl) { - this.acl = acl; - } - public long getRetryDelay() { return retryDelay; } diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java new file mode 100644 index 00000000000..b4ab6d829a5 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java @@ -0,0 +1,45 @@ +package org.apache.solr.common.cloud; + +import java.util.Collection; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public interface ZkCredentialsProvider { + + public class ZkCredentials { + String scheme; + byte[] auth; + + public ZkCredentials(String scheme, byte[] auth) { + super(); + this.scheme = scheme; + this.auth = auth; + } + + String getScheme() { + return scheme; + } + + byte[] getAuth() { + return auth; + } + } + + Collection getCredentials(); + +} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index 7ff5f364fa5..96569dc4316 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -29,13 +29,10 @@ import org.apache.commons.io.IOUtils; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs; import org.eclipse.jetty.servlet.ServletHolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.io.Files; - public class MiniSolrCloudCluster { private static Logger log = LoggerFactory.getLogger(MiniSolrCloudCluster.class); @@ -70,8 +67,7 @@ public class MiniSolrCloudCluster { AbstractZkTestCase.TIMEOUT, 45000, null); zkClient.makePath("/solr", false, true); is = new FileInputStream(solrXml); - zkClient.create("/solr/solr.xml", IOUtils.toByteArray(is), - ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true); + zkClient.create("/solr/solr.xml", IOUtils.toByteArray(is), CreateMode.PERSISTENT, true); } finally { IOUtils.closeQuietly(is); if (zkClient != null) zkClient.close();