diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8aa75277629..5b569e7973e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -198,7 +198,8 @@ Jetty 9.4.14.v20181114 Upgrade Notes ---------------------- -(No Changes) +* SOLR-12770: The 'shards' parameter handling logic changes to use a new config element to determine what hosts can be + requested. Please see Apache Solr Reference Guide chapter "Distributed Requests" for details, as well as SOLR-12770. New Features ---------------------- @@ -210,6 +211,9 @@ New Features * SOLR-11126: New Node-level health check handler at /admin/info/healthcheck and /node/health paths that checks if the node is live, connected to zookeeper and not shutdown. (Anshum Gupta, Amrit Sarkar, shalin) + +* SOLR-12770: Make it possible to configure a host whitelist for distributed search + (Christine Poerschke, janhoy, Erick Erickson, Tomás Fernández Löbbe) Bug Fixes ---------------------- diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd index 4b86d25865f..639f7e9d6a8 100755 --- a/solr/bin/solr.in.cmd +++ b/solr/bin/solr.in.cmd @@ -149,3 +149,9 @@ REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCrede REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^ REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD REM set SOLR_OPTS=%SOLR_OPTS% %SOLR_ZK_CREDS_AND_ACLS% + +REM When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the +REM list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can be configured in solr.xml, +REM or if you are using the OOTB solr.xml, can be specified using the system property "solr.shardsWhitelist". Alternatively +REM host checking can be disabled by using the system property "solr.disable.shardsWhitelist" +REM set SOLR_OPTS="%SOLR_OPTS% -Dsolr.shardsWhitelist=http://localhost:8983,http://localhost:8984" diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh index af1cd7a02c2..453796f46b7 100644 --- a/solr/bin/solr.in.sh +++ b/solr/bin/solr.in.sh @@ -178,3 +178,8 @@ #SOLR_RECOMMENDED_MAX_PROCESSES= #SOLR_ULIMIT_CHECKS= +# When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the +# list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can be configured in solr.xml, +# or if you are using the OOTB solr.xml, can be specified using the system property "solr.shardsWhitelist". Alternatively +# host checking can be disabled by using the system property "solr.disable.shardsWhitelist" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.shardsWhitelist=http://localhost:8983,http://localhost:8984" diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 1bfa6e9ab1f..03a08ce339f 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.ConnectException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -32,7 +33,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; - import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.SolrServerException; @@ -43,6 +43,7 @@ import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; @@ -89,7 +90,6 @@ public class HttpShardHandler extends ShardHandler { // This is primarily to keep track of what order we should use to query the replicas of a shard // so that we use the same replica for all phases of a distributed request. shardToURLs = new HashMap<>(); - } @@ -355,6 +355,13 @@ public class HttpShardHandler extends ShardHandler { rb.shards = new String[rb.slices.length]; } + HttpShardHandlerFactory.WhitelistHostChecker hostChecker = httpShardHandlerFactory.getWhitelistHostChecker(); + if (shards != null && zkController == null && hostChecker.isWhitelistHostCheckingEnabled() && !hostChecker.hasExplicitWhitelist()) { + throw new SolrException(ErrorCode.FORBIDDEN, "HttpShardHandlerFactory "+HttpShardHandlerFactory.INIT_SHARDS_WHITELIST + +" not configured but required (in lieu of ZkController and ClusterState) when using the '"+ShardParams.SHARDS+"' parameter." + +HttpShardHandlerFactory.SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE); + } + // // Map slices to shards // @@ -378,21 +385,32 @@ public class HttpShardHandler extends ShardHandler { if (shortCircuit) { rb.isDistrib = false; rb.shortCircuitedURL = ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), coreDescriptor.getName()); + if (hostChecker.isWhitelistHostCheckingEnabled() && hostChecker.hasExplicitWhitelist()) { + /* + * We only need to check the host whitelist if there is an explicit whitelist (other than all the live nodes) + * when the "shards" indicate cluster state elements only + */ + hostChecker.checkWhitelist(clusterState, shards, Arrays.asList(rb.shortCircuitedURL)); + } return; } // We shouldn't need to do anything to handle "shard.rows" since it was previously meant to be an optimization? } + + if (clusterState == null && zkController != null) { + clusterState = zkController.getClusterState(); + } for (int i=0; i shardUrls = StrUtils.splitSmart(rb.shards[i], "|", true); replicaListTransformer.transform(shardUrls); + hostChecker.checkWhitelist(clusterState, shards, shardUrls); // And now recreate the | delimited list of equivalent servers rb.shards[i] = createSliceShardsStr(shardUrls); } else { - if (clusterState == null) { - clusterState = zkController.getClusterState(); + if (slices == null) { slices = clusterState.getCollection(cloudDescriptor.getCollectionName()).getSlicesMap(); } String sliceName = rb.slices[i]; @@ -433,6 +451,14 @@ public class HttpShardHandler extends ShardHandler { final List shardUrls = transformReplicasToShardUrls(replicaListTransformer, eligibleSliceReplicas); + if (hostChecker.isWhitelistHostCheckingEnabled() && hostChecker.hasExplicitWhitelist()) { + /* + * We only need to check the host whitelist if there is an explicit whitelist (other than all the live nodes) + * when the "shards" indicate cluster state elements only + */ + hostChecker.checkWhitelist(clusterState, shards, shardUrls); + } + // And now recreate the | delimited list of equivalent servers final String sliceShardsStr = createSliceShardsStr(shardUrls); if (sliceShardsStr.isEmpty()) { @@ -446,6 +472,11 @@ public class HttpShardHandler extends ShardHandler { rb.shards[i] = sliceShardsStr; } } + } else { + if (shards != null) { + // No cloud, verbatim check of shards + hostChecker.checkWhitelist(shards, new ArrayList<>(Arrays.asList(shards.split("[,|]")))); + } } String shards_rows = params.get(ShardParams.SHARDS_ROWS); if(shards_rows != null) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java index 6c37e803145..3009a5b9f71 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java @@ -16,13 +16,20 @@ */ package org.apache.solr.handler.component; +import static org.apache.solr.util.stats.InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES; + +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Random; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletionService; @@ -30,7 +37,7 @@ import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; - +import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrClient; @@ -44,6 +51,8 @@ import org.apache.solr.client.solrj.impl.LBSolrClient; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ShardParams; @@ -66,8 +75,6 @@ import org.apache.solr.util.stats.MetricUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.util.stats.InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES; - public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized, SolrMetricProducer { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String DEFAULT_SCHEME = "http"; @@ -100,6 +107,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org. int permittedLoadBalancerRequestsMinimumAbsolute = 0; float permittedLoadBalancerRequestsMaximumFraction = 1.0f; boolean accessPolicy = false; + private WhitelistHostChecker whitelistHostChecker = null; private String scheme = null; @@ -133,6 +141,12 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org. // Configure if the threadpool favours fairness over throughput static final String INIT_FAIRNESS_POLICY = "fairnessPolicy"; + public static final String INIT_SHARDS_WHITELIST = "shardsWhitelist"; + + static final String INIT_SOLR_DISABLE_SHARDS_WHITELIST = "solr.disable." + INIT_SHARDS_WHITELIST; + + static final String SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE = " set -D"+INIT_SOLR_DISABLE_SHARDS_WHITELIST+"=true to disable shards whitelist checks"; + /** * Get {@link ShardHandler} that uses the default http client. */ @@ -161,6 +175,24 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org. }; } + /** + * Returns this Factory's {@link WhitelistHostChecker}. + * This method can be overridden to change the checker implementation. + */ + public WhitelistHostChecker getWhitelistHostChecker() { + return this.whitelistHostChecker; + } + + @Deprecated // For temporary use by the TermsComponent only. + static boolean doGetDisableShardsWhitelist() { + return getDisableShardsWhitelist(); + } + + + private static boolean getDisableShardsWhitelist() { + return Boolean.getBoolean(INIT_SOLR_DISABLE_SHARDS_WHITELIST); + } + @Override public void init(PluginInfo info) { StringBuilder sb = new StringBuilder(); @@ -192,6 +224,9 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org. permittedLoadBalancerRequestsMaximumFraction, sb); this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy,sb); + this.whitelistHostChecker = new WhitelistHostChecker(args == null? null: (String) args.get(INIT_SHARDS_WHITELIST), !getDisableShardsWhitelist()); + log.info("Host whitelist initialized: {}", this.whitelistHostChecker); + log.debug("created with {}",sb); // magic sysprop to make tests reproducible: set by SolrTestCaseJ4. @@ -472,5 +507,138 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org. manager.registry(registry), SolrMetricManager.mkName("httpShardExecutor", expandedScope, "threadPool")); } + + /** + * Class used to validate the hosts in the "shards" parameter when doing a distributed + * request + */ + public static class WhitelistHostChecker { + + /** + * List of the whitelisted hosts. Elements in the list will be host:port (no protocol or context) + */ + private final Set whitelistHosts; + + /** + * Indicates whether host checking is enabled + */ + private final boolean whitelistHostCheckingEnabled; + + public WhitelistHostChecker(String whitelistStr, boolean enabled) { + this.whitelistHosts = implGetShardsWhitelist(whitelistStr); + this.whitelistHostCheckingEnabled = enabled; + } + + final static Set implGetShardsWhitelist(final String shardsWhitelist) { + if (shardsWhitelist != null && !shardsWhitelist.isEmpty()) { + return StrUtils.splitSmart(shardsWhitelist, ',') + .stream() + .map(String::trim) + .map((hostUrl) -> { + URL url; + try { + if (!hostUrl.startsWith("http://") && !hostUrl.startsWith("https://")) { + // It doesn't really matter which protocol we set here because we are not going to use it. We just need a full URL. + url = new URL("http://" + hostUrl); + } else { + url = new URL(hostUrl); + } + } catch (MalformedURLException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist, e); + } + if (url.getHost() == null || url.getPort() < 0) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist); + } + return url.getHost() + ":" + url.getPort(); + }).collect(Collectors.toSet()); + } + return null; + } + + + /** + * @see #checkWhitelist(ClusterState, String, List) + */ + protected void checkWhitelist(String shardsParamValue, List shardUrls) { + checkWhitelist(null, shardsParamValue, shardUrls); + } + + /** + * Checks that all the hosts for all the shards requested in shards parameter exist in the configured whitelist + * or in the ClusterState (in case of cloud mode) + * + * @param clusterState The up to date ClusterState, can be null in case of non-cloud mode + * @param shardsParamValue The original shards parameter + * @param shardUrls The list of cores generated from the shards parameter. + */ + protected void checkWhitelist(ClusterState clusterState, String shardsParamValue, List shardUrls) { + if (!whitelistHostCheckingEnabled) { + return; + } + Set localWhitelistHosts; + if (whitelistHosts == null && clusterState != null) { + // TODO: We could implement caching, based on the version of the live_nodes znode + localWhitelistHosts = generateWhitelistFromLiveNodes(clusterState); + } else if (whitelistHosts != null) { + localWhitelistHosts = whitelistHosts; + } else { + localWhitelistHosts = Collections.emptySet(); + } + + shardUrls.stream().map(String::trim).forEach((shardUrl) -> { + URL url; + try { + if (!shardUrl.startsWith("http://") && !shardUrl.startsWith("https://")) { + // It doesn't really matter which protocol we set here because we are not going to use it. We just need a full URL. + url = new URL("http://" + shardUrl); + } else { + url = new URL(shardUrl); + } + } catch (MalformedURLException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue, e); + } + if (url.getHost() == null || url.getPort() < 0) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue); + } + if (!localWhitelistHosts.contains(url.getHost() + ":" + url.getPort())) { + log.warn("The '"+ShardParams.SHARDS+"' parameter value '"+shardsParamValue+"' contained value(s) not on the shards whitelist ("+localWhitelistHosts+"), shardUrl:" + shardUrl); + throw new SolrException(ErrorCode.FORBIDDEN, + "The '"+ShardParams.SHARDS+"' parameter value '"+shardsParamValue+"' contained value(s) not on the shards whitelist. shardUrl:" + shardUrl + "." + + HttpShardHandlerFactory.SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE); + } + }); + } + + Set generateWhitelistFromLiveNodes(ClusterState clusterState) { + return clusterState + .getLiveNodes() + .stream() + .map((liveNode) -> liveNode.substring(0, liveNode.indexOf('_'))) + .collect(Collectors.toSet()); + } + + public boolean hasExplicitWhitelist() { + return this.whitelistHosts != null; + } + + public boolean isWhitelistHostCheckingEnabled() { + return whitelistHostCheckingEnabled; + } + + /** + * Only to be used by tests + */ + @VisibleForTesting + Set getWhitelistHosts() { + return this.whitelistHosts; + } + @Override + public String toString() { + return "WhitelistHostChecker [whitelistHosts=" + whitelistHosts + ", whitelistHostCheckingEnabled=" + + whitelistHostCheckingEnabled + "]"; + } + + } + } diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java index e0949861515..1258e6043c8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java @@ -19,10 +19,10 @@ package org.apache.solr.handler.component; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.regex.Pattern; - import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -38,6 +38,8 @@ import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.mutable.MutableValue; import org.apache.solr.client.solrj.response.TermsResponse; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; @@ -45,6 +47,7 @@ import org.apache.solr.common.params.TermsParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.handler.component.HttpShardHandlerFactory.WhitelistHostChecker; import org.apache.solr.request.SimpleFacets.CountPair; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.SchemaField; @@ -76,6 +79,20 @@ public class TermsComponent extends SearchComponent { public static final int UNLIMITED_MAX_COUNT = -1; public static final String COMPONENT_NAME = "terms"; + // This needs to be created here too, because Solr doesn't call init(...) on default components. Bug? + private WhitelistHostChecker whitelistHostChecker = new WhitelistHostChecker( + null, + !HttpShardHandlerFactory.doGetDisableShardsWhitelist()); + + @Override + public void init( NamedList args ) + { + super.init(args); + whitelistHostChecker = new WhitelistHostChecker( + (String) args.get(HttpShardHandlerFactory.INIT_SHARDS_WHITELIST), + !HttpShardHandlerFactory.doGetDisableShardsWhitelist()); + } + @Override public void prepare(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); @@ -95,10 +112,30 @@ public class TermsComponent extends SearchComponent { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shards.qt parameter specified"); } List lst = StrUtils.splitSmart(shards, ",", true); + checkShardsWhitelist(rb, lst); rb.shards = lst.toArray(new String[lst.size()]); } } + protected void checkShardsWhitelist(final ResponseBuilder rb, final List lst) { + final List urls = new LinkedList(); + for (final String ele : lst) { + urls.addAll(StrUtils.splitSmart(ele, '|')); + } + + if (whitelistHostChecker.isWhitelistHostCheckingEnabled() && rb.req.getCore().getCoreContainer().getZkController() == null && !whitelistHostChecker.hasExplicitWhitelist()) { + throw new SolrException(ErrorCode.FORBIDDEN, "TermsComponent "+HttpShardHandlerFactory.INIT_SHARDS_WHITELIST + +" not configured but required when using the '"+ShardParams.SHARDS+"' parameter with the TermsComponent." + +HttpShardHandlerFactory.SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE); + } else { + ClusterState cs = null; + if (rb.req.getCore().getCoreContainer().getZkController() != null) { + cs = rb.req.getCore().getCoreContainer().getZkController().getClusterState(); + } + whitelistHostChecker.checkWhitelist(cs, urls.toString(), urls); + } + } + @Override public void process(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); @@ -687,4 +724,5 @@ public class TermsComponent extends SearchComponent { public Category getCategory() { return Category.QUERY; } + } diff --git a/solr/core/src/test-files/solr/solr.xml b/solr/core/src/test-files/solr/solr.xml index f77c9c4e008..a3dedf03f45 100644 --- a/solr/core/src/test-files/solr/solr.xml +++ b/solr/core/src/test-files/solr/solr.xml @@ -29,6 +29,7 @@ ${urlScheme:} ${socketTimeout:15000} ${connTimeout:15000} + ${solr.tests.shardsWhitelist:} diff --git a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java index 86d50a7b901..44c8bf64591 100644 --- a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java +++ b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java @@ -56,6 +56,7 @@ public class TestTolerantSearch extends SolrJettyTestBase { @BeforeClass public static void createThings() throws Exception { + systemSetPropertySolrDisableShardsWhitelist("true"); solrHome = createSolrHome(); createAndStartJetty(solrHome.getAbsolutePath()); String url = jetty.getBaseUrl().toString(); @@ -105,6 +106,7 @@ public class TestTolerantSearch extends SolrJettyTestBase { jetty.stop(); jetty=null; resetExceptionIgnores(); + systemClearPropertySolrDisableShardsWhitelist(); } @SuppressWarnings("unchecked") diff --git a/solr/core/src/test/org/apache/solr/handler/component/CustomTermsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/CustomTermsComponentTest.java new file mode 100644 index 00000000000..b0836f1273b --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/CustomTermsComponentTest.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import static org.hamcrest.CoreMatchers.containsString; + +import java.io.IOException; +import java.util.List; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.TermsResponse; +import org.apache.solr.client.solrj.response.TermsResponse.Term; +import org.apache.solr.cloud.ConfigRequest; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.common.util.NamedList; +import org.junit.Test; + +public class CustomTermsComponentTest extends ShardsWhitelistTest { + + public static class CustomTermsComponent extends TermsComponent { + + public void init( NamedList args ) + { + super.init(args); + } + + @Override + protected void checkShardsWhitelist(final ResponseBuilder rb, final List lst) { + // ignore shards whitelist + } + + } + + private static String addCustomHandlerWithTermsComponentConfig(final MiniSolrCloudCluster cluster, final String collection, + final String defaultHandlerName, final String shardsWhitelist) throws Exception { + return addCustomHandler(cluster, collection, defaultHandlerName, shardsWhitelist); + } + + private static String addCustomHandlerWithCustomTermsComponent(final MiniSolrCloudCluster cluster, final String collection, + final String defaultHandlerName) throws Exception { + return addCustomHandler(cluster, collection, defaultHandlerName, null); + } + + private static String addCustomHandler(final MiniSolrCloudCluster cluster, final String collection, + final String defaultHandlerName, final String shardsWhitelist) throws Exception { + + // determine custom handler name (the exact name should not matter) + final String customHandlerName = defaultHandlerName+"_custom"+random().nextInt(); + + // determine custom terms component name (the exact name should not matter) + final String customTermsComponentName = TermsComponent.COMPONENT_NAME+"_custom"+random().nextInt(); + + // determine terms component class name and attributes + final String customTermsComponentClass; + final String customTermsComponentAttributesJSON; + if (shardsWhitelist != null) { + customTermsComponentClass = TermsComponent.class.getName(); + customTermsComponentAttributesJSON = + " '"+HttpShardHandlerFactory.INIT_SHARDS_WHITELIST+"' : '"+shardsWhitelist+"',\n"; + } else { + customTermsComponentClass = CustomTermsComponent.class.getName(); + customTermsComponentAttributesJSON = ""; + } + + // add custom component + cluster.getSolrClient().request( + new ConfigRequest( + "{\n" + + " 'add-searchcomponent': {\n" + + customTermsComponentAttributesJSON + + " 'name': '"+customTermsComponentName+"',\n" + + " 'class': '"+customTermsComponentClass+"'\n" + + " }\n" + + "}"), + collection); + + // add custom handler + cluster.getSolrClient().request( + new ConfigRequest( + "{\n" + + " 'add-requesthandler': {\n" + + " 'name' : '"+customHandlerName+"',\n" + + " 'class' : '"+SearchHandler.class.getName()+"',\n" + + " 'components' : [ '"+QueryComponent.COMPONENT_NAME+"', '"+customTermsComponentName+"' ]\n" + + " }\n" + + "}"), + collection); + + return customHandlerName; + } + + @Test + @Override + public void test() throws Exception { + for (final String clusterId : clusterId2cluster.keySet()) { + final MiniSolrCloudCluster cluster = clusterId2cluster.get(clusterId); + final String collection = COLLECTION_NAME; + doTest(cluster, collection); + } + } + + private static void doTest(final MiniSolrCloudCluster cluster, final String collection) throws Exception { + + // add some documents + final String id = "id"; + final String f1 = "a_t"; + final String f2 = "b_t"; + final String v1 = "bee"; + final String v2 = "buzz"; + { + new UpdateRequest() + .add(sdoc(id, 1, f1, v1, f2, v2+" "+v2+" "+v2)) + .add(sdoc(id, 2, f1, v1+" "+v1, f2, v2+" "+v2)) + .add(sdoc(id, 3, f1, v1+" "+v1+" "+v1, f2, v2)) + .commit(cluster.getSolrClient(), collection); + } + + // search for the documents' terms ... + final String defaultHandlerName = "/select"; + + // search with the default handler ... + final String shards = findAndCheckTerms(cluster, collection, + defaultHandlerName, + null, // ... without specifying shards + (random().nextBoolean() ? null : f1), v1, + (random().nextBoolean() ? null : f2), v2, + null); + + // search with the default handler ... + findAndCheckTerms(cluster, collection, + defaultHandlerName, + shards, // ... with specified shards, but all valid + (random().nextBoolean() ? null : f1), v1, + (random().nextBoolean() ? null : f2), v2, + null); + + ignoreException("not on the shards whitelist"); + // this case should fail + findAndCheckTerms(cluster, collection, + defaultHandlerName, + shards + ",http://[ff01::114]:33332", // ... with specified shards with one invalid + (random().nextBoolean() ? null : f1), v1, + (random().nextBoolean() ? null : f2), v2, + "No live SolrServers available to handle this request"); + unIgnoreException("not on the shards whitelist"); + + // configure a custom handler ... + final String customHandlerName; + if (random().nextBoolean()) { + // ... with a shards whitelist + customHandlerName = addCustomHandlerWithTermsComponentConfig(cluster, collection, defaultHandlerName, shards); + } else { + // ... with a custom terms component that disregards shards whitelist logic + customHandlerName = addCustomHandlerWithCustomTermsComponent(cluster, collection, defaultHandlerName); + } + + // search with the custom handler ... + findAndCheckTerms(cluster, collection, + customHandlerName, + shards, // ... with specified shards + (random().nextBoolean() ? null : f1), v1, + (random().nextBoolean() ? null : f2), v2, + null); + + } + + private static String findAndCheckTerms(final MiniSolrCloudCluster cluster, final String collection, + String requestHandlerName, String in_shards, + String field1, String value1, + String field2, String value2, + String solrServerExceptionMessagePrefix) throws IOException { + + // compose the query ... + final SolrQuery solrQuery = new SolrQuery("*:*"); + solrQuery.setRequestHandler(requestHandlerName); + solrQuery.add("shards.qt", requestHandlerName); + // ... asking for terms ... + solrQuery.setTerms(true); + if (field1 != null) { + solrQuery.addTermsField(field1); + } + if (field2 != null) { + solrQuery.addTermsField(field2); + } + // ... and shards info ... + solrQuery.add("shards.info", "true"); + // ... passing shards to use (if we have a preference) + if (in_shards != null) { + solrQuery.add("shards", in_shards); + } + + // make the query + final QueryResponse queryResponse; + try { + queryResponse = new QueryRequest(solrQuery) + .process(cluster.getSolrClient(), collection); + assertNull("expected exception ("+solrServerExceptionMessagePrefix+") not encountered", solrServerExceptionMessagePrefix); + } catch (SolrServerException sse) { + assertNotNull("unexpectedly caught exception "+sse, solrServerExceptionMessagePrefix); + assertTrue(sse.getMessage().startsWith(solrServerExceptionMessagePrefix)); + assertThat(sse.getCause().getMessage(), containsString("not on the shards whitelist")); + return null; + } + + // analyse the response ... + final TermsResponse termsResponse = queryResponse.getTermsResponse(); + // ... checking the terms returned ... + checkTermsResponse(termsResponse, field1, value1); + checkTermsResponse(termsResponse, field2, value2); + // ... and assemble info about the shards ... + final String out_shards = extractShardAddresses(queryResponse, ","); + // ... to return to the caller + return out_shards; + } + + + @SuppressWarnings("unchecked") + private static String extractShardAddresses(final QueryResponse queryResponse, final String delimiter) { + final StringBuilder sb = new StringBuilder(); + final NamedList nl = (NamedList)queryResponse.getResponse().get("shards.info"); + assertNotNull(queryResponse.toString(), nl); + for (int ii = 0; ii < nl.size(); ++ii) { + final String shardAddress = (String)((NamedList)nl.getVal(ii)).get("shardAddress"); + if (sb.length() > 0) { + sb.append(delimiter); + } + sb.append(shardAddress); + } + return sb.toString(); + } + + private static void checkTermsResponse(TermsResponse termsResponse, String field, String value) { + if (field != null) { + final List ttList = termsResponse.getTerms(field); + assertEquals(1, ttList.size()); + assertEquals(value, ttList.get(0).getTerm()); + } + } + +} diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java index f1daf395e31..5d3c7f414f1 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java @@ -63,6 +63,7 @@ public class DistributedDebugComponentTest extends SolrJettyTestBase { @BeforeClass public static void createThings() throws Exception { + systemSetPropertySolrDisableShardsWhitelist("true"); solrHome = createSolrHome(); createAndStartJetty(solrHome.getAbsolutePath()); String url = jetty.getBaseUrl().toString(); @@ -105,6 +106,7 @@ public class DistributedDebugComponentTest extends SolrJettyTestBase { jetty.stop(); jetty=null; resetExceptionIgnores(); + systemClearPropertySolrDisableShardsWhitelist(); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/component/ShardsWhitelistTest.java b/solr/core/src/test/org/apache/solr/handler/component/ShardsWhitelistTest.java new file mode 100644 index 00000000000..8aea6ebda87 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/ShardsWhitelistTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.cloud.MultiSolrCloudTestCase; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ShardsWhitelistTest extends MultiSolrCloudTestCase { + + /** + * The cluster with this key will include an explicit list of host whitelisted (all hosts in both the clusters) + */ + private static final String EXPLICIT_CLUSTER_KEY = "explicitCluster"; + /** + * The cluster with this key will not include an explicit list of host whitelisted, will rely on live_nodes + */ + private static final String IMPLICIT_CLUSTER_KEY = "implicitCluster"; + private static final String EXPLICIT_WHITELIST_PROPERTY = "solr.tests.ShardsWhitelistTest.explicitWhitelist."; + protected static final String COLLECTION_NAME = "ShardsWhitelistTestCollection"; + + private static int numShards; + private static int numReplicas; + private static int maxShardsPerNode; + private static int nodesPerCluster; + + private static void appendClusterNodes(final StringBuilder sb, final String delimiter, + final MiniSolrCloudCluster cluster) { + cluster.getJettySolrRunners().forEach((jetty) -> sb.append(jetty.getBaseUrl().toString() + delimiter)); + } + + @BeforeClass + public static void setupClusters() throws Exception { + + final String[] clusterIds = new String[] {IMPLICIT_CLUSTER_KEY, EXPLICIT_CLUSTER_KEY}; + + numShards = 2; // +random().nextInt(2); + numReplicas = 1; // +random().nextInt(2); + maxShardsPerNode = 1; // +random().nextInt(2); + nodesPerCluster = (numShards * numReplicas + (maxShardsPerNode - 1)) / maxShardsPerNode; + + final StringBuilder sb = new StringBuilder(); + + doSetupClusters(clusterIds, + new DefaultClusterCreateFunction() { + + @Override + public MiniSolrCloudCluster apply(String clusterId) { + try { + final MiniSolrCloudCluster cluster = new SolrCloudTestCase.Builder(nodesPerCluster(clusterId), + createTempDir()) + .addConfig("conf", configset("cloud-dynamic")) + .withSolrXml(MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML.replace( + MiniSolrCloudCluster.SOLR_TESTS_SHARDS_WHITELIST, EXPLICIT_WHITELIST_PROPERTY + clusterId)) + .build(); + return cluster; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected int nodesPerCluster(String clusterId) { + return nodesPerCluster; + } + }, + new DefaultClusterInitFunction(numShards, numReplicas, maxShardsPerNode) { + @Override + public void accept(String clusterId, MiniSolrCloudCluster cluster) { + appendClusterNodes(sb, ",", cluster); + if (clusterId.equals(EXPLICIT_CLUSTER_KEY)) { + System.setProperty(EXPLICIT_WHITELIST_PROPERTY + clusterId, sb.toString()); + for (JettySolrRunner runner : cluster.getJettySolrRunners()) { + try { + runner.stop(); + runner.start(true); + } catch (Exception e) { + throw new RuntimeException("Unable to restart runner", e); + } + } + } + doAccept(COLLECTION_NAME, cluster); + } + }); + } + + @AfterClass + public static void afterTests() { + System.clearProperty(EXPLICIT_WHITELIST_PROPERTY + EXPLICIT_CLUSTER_KEY); + } + + private HttpShardHandlerFactory getShardHandlerFactory(String clusterId) { + return (HttpShardHandlerFactory) clusterId2cluster.get(clusterId).getJettySolrRunner(0).getCoreContainer() + .getShardHandlerFactory(); + } + + @Test + public void test() throws Exception { + assertThat(getShardHandlerFactory(EXPLICIT_CLUSTER_KEY).getWhitelistHostChecker().getWhitelistHosts(), notNullValue()); + assertThat(getShardHandlerFactory(IMPLICIT_CLUSTER_KEY).getWhitelistHostChecker().getWhitelistHosts(), nullValue()); + + assertThat(getShardHandlerFactory(EXPLICIT_CLUSTER_KEY).getWhitelistHostChecker().hasExplicitWhitelist(), is(true)); + assertThat(getShardHandlerFactory(IMPLICIT_CLUSTER_KEY).getWhitelistHostChecker().hasExplicitWhitelist(), is(false)); + for (MiniSolrCloudCluster cluster : clusterId2cluster.values()) { + for (JettySolrRunner runner : cluster.getJettySolrRunners()) { + URI uri = runner.getBaseUrl().toURI(); + assertThat(getShardHandlerFactory(EXPLICIT_CLUSTER_KEY).getWhitelistHostChecker().getWhitelistHosts(), + hasItem(uri.getHost() + ":" + uri.getPort())); + } + } + + MiniSolrCloudCluster implicitCluster = clusterId2cluster.get(IMPLICIT_CLUSTER_KEY); + MiniSolrCloudCluster explicitCluster = clusterId2cluster.get(EXPLICIT_CLUSTER_KEY); + + for (Map.Entry entry : clusterId2cluster.entrySet()) { + List docs = new ArrayList<>(10); + for (int i = 0; i < 10; i++) { + docs.add(new SolrInputDocument("id", entry.getKey() + i)); + } + MiniSolrCloudCluster cluster = entry.getValue(); + cluster.getSolrClient().add(COLLECTION_NAME, docs); + cluster.getSolrClient().commit(COLLECTION_NAME, true, true); + + // test using ClusterState elements + assertThat("No shards specified, should work in both clusters", + numDocs("*:*", null, cluster), is(10)); + assertThat("Both shards specified, should work in both clusters", + numDocs("*:*", "shard1,shard2", cluster), is(10)); + assertThat("Both shards specified with collection name, should work in both clusters", + numDocs("*:*", COLLECTION_NAME + "_shard1", cluster), is(numDocs("*:*", "shard1", cluster))); + + // test using explicit urls from within the cluster + assertThat("Shards has the full URLs, should be allowed since they are internal. Cluster=" + entry.getKey(), + numDocs("*:*", getShardUrl("shard1", cluster) + "," + getShardUrl("shard2", cluster), cluster), is(10)); + assertThat("Full URL without scheme", + numDocs("*:*", getShardUrl("shard1", cluster).replaceAll("http://", "") + "," + + getShardUrl("shard2", cluster).replaceAll("http://", ""), cluster), + is(10)); + + // Mix shards with URLs + assertThat("Mix URL and cluster state object", + numDocs("*:*", "shard1," + getShardUrl("shard2", cluster), cluster), is(10)); + assertThat("Mix URL and cluster state object", + numDocs("*:*", getShardUrl("shard1", cluster) + ",shard2", cluster), is(10)); + } + + // explicit whitelist includes all the nodes in both clusters. Requests should be allowed to go through + assertThat("A request to the explicit cluster with shards that point to the implicit one", + numDocs( + "id:implicitCluster*", + getShardUrl("shard1", implicitCluster) + "," + getShardUrl("shard2", implicitCluster), + explicitCluster), + is(10)); + + assertThat("A request to the explicit cluster with shards that point to the both clusters", + numDocs( + "*:*", + getShardUrl("shard1", implicitCluster) + + "," + getShardUrl("shard2", implicitCluster) + + "," + getShardUrl("shard1", explicitCluster) + + "," + getShardUrl("shard2", explicitCluster), + explicitCluster), + is(20)); + + // Implicit shouldn't allow requests to the other cluster + assertForbidden("id:explicitCluster*", + getShardUrl("shard1", explicitCluster) + "," + getShardUrl("shard2", explicitCluster), + implicitCluster); + + assertForbidden("id:explicitCluster*", + "shard1," + getShardUrl("shard2", explicitCluster), + implicitCluster); + + assertForbidden("id:explicitCluster*", + getShardUrl("shard1", explicitCluster) + ",shard2", + implicitCluster); + + assertForbidden("id:explicitCluster*", + getShardUrl("shard1", explicitCluster), + implicitCluster); + + assertThat("A typical internal request, should be handled locally", + numDocs( + "id:explicitCluster*", + null, + implicitCluster, + "distrib", "false", + "shard.url", getShardUrl("shard2", explicitCluster), + "shards.purpose", "64", + "isShard", "true"), + is(0)); + } + + private void assertForbidden(String query, String shards, MiniSolrCloudCluster cluster) throws IOException { + ignoreException("not on the shards whitelist"); + try { + numDocs( + query, + shards, + cluster); + fail("Expecting failure for shards parameter: '" + shards + "'"); + } catch (SolrServerException e) { + assertThat(e.getCause(), instanceOf(SolrException.class)); + assertThat(((SolrException) e.getCause()).code(), is(SolrException.ErrorCode.FORBIDDEN.code)); + assertThat(((SolrException) e.getCause()).getMessage(), containsString("not on the shards whitelist")); + } + unIgnoreException("not on the shards whitelist"); + } + + private String getShardUrl(String shardName, MiniSolrCloudCluster cluster) { + return cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME) + .getSlice(shardName).getReplicas().iterator().next().getCoreUrl(); + } + + private int numDocs(String queryString, String shardsParamValue, MiniSolrCloudCluster cluster, String... otherParams) + throws SolrServerException, IOException { + SolrQuery q = new SolrQuery(queryString); + if (shardsParamValue != null) { + q.set("shards", shardsParamValue); + } + if (otherParams != null) { + assert otherParams.length % 2 == 0; + for (int i = 0; i < otherParams.length; i += 2) { + q.set(otherParams[i], otherParams[i + 1]); + } + } + return (int) cluster.getSolrClient().query(COLLECTION_NAME, q).getResults().getNumFound(); + } + +} diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java index 34fc69f4d76..2ee73cb02bf 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java +++ b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java @@ -16,22 +16,34 @@ */ package org.apache.solr.handler.component; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; + import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; - +import java.util.Set; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.impl.LBSolrClient; import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; - -import org.junit.BeforeClass; +import org.apache.solr.handler.component.HttpShardHandlerFactory.WhitelistHostChecker; import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; /** * Tests specifying a custom ShardHandlerFactory @@ -40,6 +52,7 @@ public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 { private static final String LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE = "solr.tests.loadBalancerRequestsMinimumAbsolute"; private static final String LOAD_BALANCER_REQUESTS_MAX_FRACTION = "solr.tests.loadBalancerRequestsMaximumFraction"; + private static final String SHARDS_WHITELIST = "solr.tests.shardsWhitelist"; private static int expectedLoadBalancerRequestsMinimumAbsolute = 0; private static float expectedLoadBalancerRequestsMaximumFraction = 1.0f; @@ -50,6 +63,7 @@ public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 { expectedLoadBalancerRequestsMaximumFraction = (1+random().nextInt(10))/10f; // 0.1 .. 1.0 System.setProperty(LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE, Integer.toString(expectedLoadBalancerRequestsMinimumAbsolute)); System.setProperty(LOAD_BALANCER_REQUESTS_MAX_FRACTION, Float.toString(expectedLoadBalancerRequestsMaximumFraction)); + } @AfterClass @@ -216,4 +230,186 @@ public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 { } } + @Test + public void getShardsWhitelist() throws Exception { + System.setProperty(SHARDS_WHITELIST, "http://abc:8983/,http://def:8984/,"); + final Path home = Paths.get(TEST_HOME()); + CoreContainer cc = null; + ShardHandlerFactory factory = null; + try { + cc = CoreContainer.createAndLoad(home, home.resolve("solr.xml")); + factory = cc.getShardHandlerFactory(); + assertTrue(factory instanceof HttpShardHandlerFactory); + final HttpShardHandlerFactory httpShardHandlerFactory = ((HttpShardHandlerFactory)factory); + assertThat(httpShardHandlerFactory.getWhitelistHostChecker().getWhitelistHosts().size(), is(2)); + assertThat(httpShardHandlerFactory.getWhitelistHostChecker().getWhitelistHosts(), hasItem("abc:8983")); + assertThat(httpShardHandlerFactory.getWhitelistHostChecker().getWhitelistHosts(), hasItem("def:8984")); + } finally { + if (factory != null) factory.close(); + if (cc != null) cc.shutdown(); + System.clearProperty(SHARDS_WHITELIST); + } + } + + @Test + public void testLiveNodesToHostUrl() throws Exception { + Set liveNodes = new HashSet<>(Arrays.asList(new String[]{ + "1.2.3.4:8983_solr", + "1.2.3.4:9000_", + "1.2.3.4:9001_solr-2", + })); + ClusterState cs = new ClusterState(0, liveNodes, new HashMap<>()); + WhitelistHostChecker checker = new WhitelistHostChecker(null, true); + Set hostSet = checker.generateWhitelistFromLiveNodes(cs); + assertThat(hostSet.size(), is(3)); + assertThat(hostSet, hasItem("1.2.3.4:8983")); + assertThat(hostSet, hasItem("1.2.3.4:9000")); + assertThat(hostSet, hasItem("1.2.3.4:9001")); + } + + @Test + public void testWhitelistHostCheckerDisabled() throws Exception { + WhitelistHostChecker checker = new WhitelistHostChecker("http://cde:8983", false); + checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"abc-1.com:8983/solr"})); + + try { + checker = new WhitelistHostChecker("http://cde:8983", true); + checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr"})); + fail("Expecting exception"); + } catch (SolrException se) { + assertThat(se.code(), is(SolrException.ErrorCode.FORBIDDEN.code)); + } + } + + @Test + public void testWhitelistHostCheckerNoInput() throws Exception { + assertNull("Whitelist hosts should be null with null input", + new WhitelistHostChecker(null, true).getWhitelistHosts()); + assertNull("Whitelist hosts should be null with empty input", + new WhitelistHostChecker("", true).getWhitelistHosts()); + } + + @Test + public void testWhitelistHostCheckerSingleHost() { + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983/solr", true); + checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr"})); + } + + @Test + public void testWhitelistHostCheckerMultipleHost() { + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true); + checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr"})); + } + + @Test + public void testWhitelistHostCheckerMultipleHost2() { + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true); + checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr", "http://abc-2.com:8983/solr"})); + } + + @Test + public void testWhitelistHostCheckerNoProtocolInParameter() { + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true); + checker.checkWhitelist("abc-1.com:8983/solr", Arrays.asList(new String[]{"abc-1.com:8983/solr"})); + } + + @Test + public void testWhitelistHostCheckerNonWhitelistedHost1() { + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true); + try { + checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-4.com:8983/solr"})); + fail("Expected exception"); + } catch (SolrException e) { + assertThat(e.code(), is(SolrException.ErrorCode.FORBIDDEN.code)); + assertThat(e.getMessage(), containsString("not on the shards whitelist")); + } + } + + @Test + public void testWhitelistHostCheckerNonWhitelistedHost2() { + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true); + try { + checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr", "http://abc-4.com:8983/solr"})); + fail("Expected exception"); + } catch (SolrException e) { + assertThat(e.code(), is(SolrException.ErrorCode.FORBIDDEN.code)); + assertThat(e.getMessage(), containsString("not on the shards whitelist")); + } + } + + @Test + public void testWhitelistHostCheckerNonWhitelistedHostHttps() { + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true); + checker.checkWhitelist("https://abc-1.com:8983/solr", Arrays.asList(new String[]{"https://abc-1.com:8983/solr"})); + } + + @Test + public void testWhitelistHostCheckerInvalidUrl() { + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true); + try { + checker.checkWhitelist("abc_1", Arrays.asList(new String[]{"abc_1"})); + fail("Expected exception"); + } catch (SolrException e) { + assertThat(e.code(), is(SolrException.ErrorCode.BAD_REQUEST.code)); + assertThat(e.getMessage(), containsString("Invalid URL syntax")); + } + } + + @Test + public void testWhitelistHostCheckerCoreSpecific() { + // cores are removed completely so it doesn't really matter if they were set in config + WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983/solr/core1, http://abc-2.com:8983/solr2/core2", true); + checker.checkWhitelist("http://abc-1.com:8983/solr/core2", Arrays.asList(new String[]{"http://abc-1.com:8983/solr/core2"})); + } + + @Test + public void testGetShardsOfWhitelistedHostsUnset() { + assertThat(WhitelistHostChecker.implGetShardsWhitelist(null), nullValue()); + } + + @Test + public void testGetShardsOfWhitelistedHostsEmpty() { + assertThat(WhitelistHostChecker.implGetShardsWhitelist(""), nullValue()); + } + + @Test + public void testGetShardsOfWhitelistedHostsSingle() { + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1").size(), is(1)); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1").iterator().next(), equalTo("abc-1.com:8983")); + } + + @Test + public void testGetShardsOfWhitelistedHostsMulti() { + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1,http://abc-1.com:8984/solr").size(), is(2)); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1,http://abc-1.com:8984/solr"), hasItem("abc-1.com:8983")); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1,http://abc-1.com:8984/solr"), hasItem("abc-1.com:8984")); + } + + @Test + public void testGetShardsOfWhitelistedHostsIpv4() { + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://10.0.0.1:8983/solr/core1,http://127.0.0.1:8984/solr").size(), is(2)); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://10.0.0.1:8983/solr/core1,http://127.0.0.1:8984/solr"), hasItem("10.0.0.1:8983")); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://10.0.0.1:8983/solr/core1,http://127.0.0.1:8984/solr"), hasItem("127.0.0.1:8984")); + } + + @Test + public void testGetShardsOfWhitelistedHostsIpv6() { + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://[2001:abc:abc:0:0:123:456:1234]:8983/solr/core1,http://[::1]:8984/solr").size(), is(2)); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://[2001:abc:abc:0:0:123:456:1234]:8983/solr/core1,http://[::1]:8984/solr"), hasItem("[2001:abc:abc:0:0:123:456:1234]:8983")); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://[2001:abc:abc:0:0:123:456:1234]:8983/solr/core1,http://[::1]:8984/solr"), hasItem("[::1]:8984")); + } + + @Test + public void testGetShardsOfWhitelistedHostsHttps() { + assertThat(WhitelistHostChecker.implGetShardsWhitelist("https://abc-1.com:8983/solr/core1").size(), is(1)); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("https://abc-1.com:8983/solr/core1"), hasItem("abc-1.com:8983")); + } + + @Test + public void testGetShardsOfWhitelistedHostsNoProtocol() { + assertThat(WhitelistHostChecker.implGetShardsWhitelist("abc-1.com:8983/solr"), + equalTo(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr"))); + assertThat(WhitelistHostChecker.implGetShardsWhitelist("abc-1.com:8983/solr"), + equalTo(WhitelistHostChecker.implGetShardsWhitelist("https://abc-1.com:8983/solr"))); + } } diff --git a/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java b/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java index 0bf46e756d3..765e9841d12 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java +++ b/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java @@ -42,6 +42,7 @@ public class TestSmileRequest extends SolrTestCaseJ4 { @BeforeClass public static void beforeTests() throws Exception { + systemSetPropertySolrDisableShardsWhitelist("true"); JSONTestUtil.failRepeatedKeys = true; initCore("solrconfig-tlog.xml", "schema_latest.xml"); } @@ -59,6 +60,7 @@ public class TestSmileRequest extends SolrTestCaseJ4 { servers.stop(); servers = null; } + systemClearPropertySolrDisableShardsWhitelist(); } @Test diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java index e757b665d77..a3339140a4c 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java @@ -41,6 +41,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS { @BeforeClass public static void beforeTests() throws Exception { + systemSetPropertySolrDisableShardsWhitelist("true"); // we need DVs on point fields to compute stats & facets if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true"); @@ -61,6 +62,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS { servers.stop(); servers = null; } + systemClearPropertySolrDisableShardsWhitelist(); } diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java index 89098150cab..0eccea80a66 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java @@ -16,6 +16,8 @@ */ package org.apache.solr.search.facet; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.tdunning.math.stats.AVLTreeDigest; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -26,21 +28,17 @@ import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; - -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.tdunning.math.stats.AVLTreeDigest; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.common.SolrException; -import org.apache.solr.util.hll.HLL; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.JSONTestUtil; import org.apache.solr.SolrTestCaseHS; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.macro.MacroExpander; - +import org.apache.solr.util.hll.HLL; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -73,14 +71,37 @@ public class TestJsonFacets extends SolrTestCaseHS { initCore("solrconfig-tlog.xml","schema_latest.xml"); } + /** + * Start all servers for cluster, initialize shards whitelist and then restart + */ public static void initServers() throws Exception { if (servers == null) { servers = new SolrInstances(3, "solrconfig-tlog.xml", "schema_latest.xml"); + // Set the shards whitelist to all shards plus the fake one used for tolerant test + System.setProperty(SOLR_TESTS_SHARDS_WHITELIST, servers.getWhitelistString() + ",http://[ff01::114]:33332"); + systemSetPropertySolrDisableShardsWhitelist("false"); + restartServers(); } } + /** + * Restart all configured servers, i.e. configuration will be re-read + */ + public static void restartServers() { + servers.slist.forEach(s -> { + try { + s.stop(); + s.start(); + } catch (Exception e) { + fail("Exception during server restart: " + e.getMessage()); + } + }); + } + @AfterClass public static void afterTests() throws Exception { + System.clearProperty(SOLR_TESTS_SHARDS_WHITELIST); + systemClearPropertySolrDisableShardsWhitelist(); JSONTestUtil.failRepeatedKeys = false; FacetFieldProcessorByHashDV.MAXIMUM_STARTING_TABLE_SIZE=origTableSize; FacetField.FacetMethod.DEFAULT_METHOD = origDefaultFacetMethod; @@ -2318,6 +2339,7 @@ public class TestJsonFacets extends SolrTestCaseHS { public void doTestPrelimSortingDistrib(final boolean extraAgg, final boolean extraSubFacet) throws Exception { // we only use 2 shards, but we also want to to sanity check code paths if one (additional) shard is empty final int totalShards = random().nextBoolean() ? 2 : 3; + final SolrInstances nodes = new SolrInstances(totalShards, "solrconfig-tlog.xml", "schema_latest.xml"); try { final Client client = nodes.getClient(random().nextInt()); @@ -2789,7 +2811,7 @@ public class TestJsonFacets extends SolrTestCaseHS { public void testTolerant() throws Exception { initServers(); Client client = servers.getClient(random().nextInt()); - client.queryDefaults().set("shards", servers.getShards() + ",[ff01::114]:33332:/ignore_exception"); + client.queryDefaults().set("shards", servers.getShards() + ",[ff01::114]:33332/ignore_exception"); indexSimple(client); try { diff --git a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java index 6efe840633b..bcc936d5378 100644 --- a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java +++ b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java @@ -32,6 +32,7 @@ public class TestJsonRequest extends SolrTestCaseHS { @BeforeClass public static void beforeTests() throws Exception { + systemSetPropertySolrDisableShardsWhitelist("true"); JSONTestUtil.failRepeatedKeys = true; initCore("solrconfig-tlog.xml","schema_latest.xml"); } @@ -49,6 +50,7 @@ public class TestJsonRequest extends SolrTestCaseHS { servers.stop(); servers = null; } + systemClearPropertySolrDisableShardsWhitelist(); } @Test diff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xml index 68b15ba0116..795e352831f 100644 --- a/solr/server/solr/solr.xml +++ b/solr/server/solr/solr.xml @@ -48,6 +48,7 @@ class="HttpShardHandlerFactory"> ${socketTimeout:600000} ${connTimeout:60000} + ${solr.shardsWhitelist:} diff --git a/solr/solr-ref-guide/src/distributed-requests.adoc b/solr/solr-ref-guide/src/distributed-requests.adoc index 56c2d1d48cb..b5246f2b763 100644 --- a/solr/solr-ref-guide/src/distributed-requests.adoc +++ b/solr/solr-ref-guide/src/distributed-requests.adoc @@ -85,7 +85,7 @@ To configure the standard search handler, provide a configuration like this in ` ---- -The parameters that can be specified are as follows: +`HttpShardHandlerFactory` is the only `ShardHandlerFactory` implementation included out of the box with Solr, It accepts the following parameters: `socketTimeout`:: The amount of time in ms that a socket is allowed to wait. The default is `0`, where the operating system's default will be used. @@ -111,6 +111,12 @@ If specified, the thread pool will use a backing queue instead of a direct hando `fairnessPolicy`:: Chooses the JVM specifics dealing with fair policy queuing, if enabled distributed searches will be handled in a First in First out fashion at a cost to throughput. If disabled throughput will be favored over latency. The default is `false`. +`shardsWhitelist`:: +If specified, this lists limits what nodes can be requested in the `shards` request parameter. In cloud mode this whitelist is automatically configured to include all live nodes in the cluster. In standalone mode the whitelist defaults to empty (sharding not allowed). If you need to disable this feature for backwards compatibility, you can set the system property `solr.disable.shardsWhitelist=true`. The value of this parameter is a comma separated list of the nodes that will be whitelisted, i.e.: +`10.0.0.1:8983/solr,10.0.0.1:8984/solr`. + +NOTE: In cloud mode, if at least one node is included in the whitelist, then the live_nodes will no longer be used as source for the list. This means that, if you need to do a cross-cluster request using the `shards` parameter in cloud mode (in addition to regular within-cluster requests), you'll need to add all nodes (local cluster + remote nodes) to the whitelist. + == Configuring statsCache (Distributed IDF) Document and term statistics are needed in order to calculate relevancy. Solr provides four implementations out of the box when it comes to document stats calculation: diff --git a/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc b/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc index 70324acdcd5..d8a2d76a317 100644 --- a/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc +++ b/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc @@ -59,6 +59,9 @@ The following components support distributed search: * The *Stats* component, which returns simple statistics for numeric fields within the DocSet. * The *Debug* component, which helps with debugging. +=== Shards Whitelist +What nodes are allowed in the `shards` parameter is configurable through the `shardsWhitelist` property in `solr.xml`. This whitelist is automatically configured for SolrCloud but needs explicit configuration for master/slave mode. Read more details in <>. + == Limitations to Distributed Search Distributed searching in Solr has the following limitations: diff --git a/solr/solr-ref-guide/src/the-terms-component.adoc b/solr/solr-ref-guide/src/the-terms-component.adoc index 6fe40d93d6b..b2705b1c0c4 100644 --- a/solr/solr-ref-guide/src/the-terms-component.adoc +++ b/solr/solr-ref-guide/src/the-terms-component.adoc @@ -295,3 +295,5 @@ Specifies the shards in your distributed indexing configuration. For more inform `shards.qt`:: Specifies the request handler Solr uses for requests to shards. + +Same as with regular distributed search, the `shards` parameter is subject to a host whitelist that has to be configured in the component init parameters using the configuration key `shardsWhitelist` and the list of hosts as values. In the same way as with distributed search, the whitelist will be populated to all live nodes by default when running in SolrCloud mode. If you need to disable this feature for backwards compatibility, you can set the system property `solr.disable.shardsWhitelist=true`. diff --git a/solr/solrj/src/test-files/solrj/solr/solr.xml b/solr/solrj/src/test-files/solrj/solr/solr.xml index 6eef53fbf05..0e9f3f4109b 100644 --- a/solr/solrj/src/test-files/solrj/solr/solr.xml +++ b/solr/solrj/src/test-files/solrj/solr/solr.xml @@ -30,6 +30,7 @@ ${urlScheme:} ${socketTimeout:90000} ${connTimeout:15000} + ${solr.tests.shardsWhitelist:} diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java index 5e4cab2728c..a881084f75d 100644 --- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java @@ -174,6 +174,18 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 { System.clearProperty("hostContext"); } + @SuppressWarnings("deprecation") + @BeforeClass + public static void setSolrDisableShardsWhitelist() throws Exception { + systemSetPropertySolrDisableShardsWhitelist("true"); + } + + @SuppressWarnings("deprecation") + @AfterClass + public static void clearSolrDisableShardsWhitelist() throws Exception { + systemClearPropertySolrDisableShardsWhitelist(); + } + private static String getHostContextSuitableForServletContext() { String ctx = System.getProperty("hostContext","/solr"); if ("".equals(ctx)) ctx = "/solr"; diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java index 32ef9a7c025..0e7edfe4371 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java @@ -64,6 +64,8 @@ import org.slf4j.LoggerFactory; //@LuceneTestCase.SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Lucene45","Appending","Asserting"}) public class SolrTestCaseHS extends SolrTestCaseJ4 { + public static final String SOLR_TESTS_SHARDS_WHITELIST = "solr.tests.shardsWhitelist"; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @SafeVarargs public static Set set(T... a) { @@ -468,6 +470,12 @@ public class SolrTestCaseHS extends SolrTestCaseJ4 { // silly stuff included from solrconfig.snippet.randomindexconfig.xml System.setProperty("solr.tests.maxBufferedDocs", String.valueOf(100000)); + + // If we want to run with whitelist list, this must be explicitly set to true for the test + // otherwise we disable the check + if (System.getProperty(SYSTEM_PROPERTY_SOLR_DISABLE_SHARDS_WHITELIST) == null) { + systemSetPropertySolrDisableShardsWhitelist("true"); + } jetty.start(); port = jetty.getLocalPort(); @@ -534,6 +542,20 @@ public class SolrTestCaseHS extends SolrTestCaseJ4 { public String getShards() { return getShardsParam(slist); } + + public String getWhitelistString() { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (SolrInstance instance : slist) { + if (first) { + first = false; + } else { + sb.append(','); + } + sb.append( instance.getBaseURL().replace("/solr", "")); + } + return sb.toString(); + } public List getSolrJs() { List solrjs = new ArrayList<>(slist.size()); diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 3145861d6b2..748721d0ac8 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -191,6 +191,9 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { public static final String SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICYFACTORY = "solr.tests.mergePolicyFactory"; + @Deprecated // For backwards compatibility only. Please do not use in new tests. + public static final String SYSTEM_PROPERTY_SOLR_DISABLE_SHARDS_WHITELIST = "solr.disable.shardsWhitelist"; + protected static String coreName = DEFAULT_TEST_CORENAME; public static int DEFAULT_CONNECTION_TIMEOUT = 60000; // default socket connection timeout in ms @@ -2718,6 +2721,16 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { System.clearProperty(SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICYFACTORY); } + @Deprecated // For backwards compatibility only. Please do not use in new tests. + protected static void systemSetPropertySolrDisableShardsWhitelist(String value) { + System.setProperty(SYSTEM_PROPERTY_SOLR_DISABLE_SHARDS_WHITELIST, value); + } + + @Deprecated // For backwards compatibility only. Please do not use in new tests. + protected static void systemClearPropertySolrDisableShardsWhitelist() { + System.clearProperty(SYSTEM_PROPERTY_SOLR_DISABLE_SHARDS_WHITELIST); + } + protected T pickRandom(T... options) { return options[random().nextInt(options.length)]; } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index fd719edad33..2fbeba812c5 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -82,6 +82,8 @@ public class MiniSolrCloudCluster { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String SOLR_TESTS_SHARDS_WHITELIST = "solr.tests.shardsWhitelist"; + public static final String DEFAULT_CLOUD_SOLR_XML = "\n" + "\n" + " ${shareSchema:false}\n" + @@ -93,6 +95,7 @@ public class MiniSolrCloudCluster { " ${urlScheme:}\n" + " ${socketTimeout:90000}\n" + " ${connTimeout:15000}\n" + + " ${"+SOLR_TESTS_SHARDS_WHITELIST+":}\n" + " \n" + "\n" + " \n" +