Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-13865: Migrate replica routing code to SolrJ (#974) 

* [SOLR-13865] Migrate replica routing code to  solrJ

* Added a CommonTestInjection class.

* Fixing imports.

* Reverted extraneous streaming changes.

* Fix precommit errors.

* Changing name of the RLTManager.

* Splitting up existing tests.

* Updated documentation.

* Added solr/CHANGES.txt entry
This commit is contained in:
Houston Putman 2019-10-28 18:49:21 -04:00 committed by Tomas Fernandez Lobbe
parent 3af4e6adc6
commit fa27e476f7
22 changed files with 790 additions and 570 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -113,6 +113,8 @@ Improvements
* SOLR-13831: Support defining arbitrary autoscaling simulation scenarios. (ab)
* SOLR-13865: Move replica routing code to SolrJ. (Houston Putman via Tomas Fernandez-Lobbe)
Optimizations
---------------------

1
solr/core/src/java/org/apache/solr/cloud/ZkController.java
View File

@ -75,6 +75,7 @@ import org.apache.solr.common.cloud.DefaultZkCredentialsProvider;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocCollectionWatcher;
import org.apache.solr.common.cloud.LiveNodesListener;
import org.apache.solr.common.cloud.NodesSysPropsCacher;
import org.apache.solr.common.cloud.OnReconnect;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Replica.Type;

4
solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
View File

@ -38,6 +38,7 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.CommonTestInjection;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
@ -48,7 +49,6 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
import org.apache.solr.util.TestInjection;
import org.apache.solr.util.stats.MetricUtils;
/**
@ -70,7 +70,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
private CoreContainer cc;
private final Map<String, String> injectedSysProps = TestInjection.injectAdditionalProps();
private final Map<String, String> injectedSysProps = CommonTestInjection.injectAdditionalProps();
public MetricsHandler() {
this.metricManager = null;

1
solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
View File

@ -43,6 +43,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.LBSolrClient;
import org.apache.solr.client.solrj.routing.ReplicaListTransformer;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.CloudDescriptor;

357
solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
View File

@ -16,20 +16,13 @@
*/
package org.apache.solr.handler.component;
import static org.apache.solr.util.stats.InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
@ -41,6 +34,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrClient;
@ -51,16 +46,16 @@ import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
import org.apache.solr.client.solrj.impl.LBSolrClient;
import org.apache.solr.client.solrj.impl.PreferenceRule;
import org.apache.solr.client.solrj.routing.AffinityReplicaListTransformerFactory;
import org.apache.solr.client.solrj.routing.ReplicaListTransformer;
import org.apache.solr.client.solrj.routing.ReplicaListTransformerFactory;
import org.apache.solr.client.solrj.routing.RequestReplicaListTransformerGenerator;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.NodesSysPropsCacher;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
@ -83,10 +78,12 @@ import org.apache.solr.util.stats.MetricUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.util.stats.InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES;
public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized, SolrMetricProducer {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String DEFAULT_SCHEME = "http";
// We want an executor that doesn't take up any resources if
// it's not used, so it could be created statically for
// the distributed search component if desired.
@ -124,7 +121,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
protected final Random r = new Random();
private final ReplicaListTransformer shufflingReplicaListTransformer = new ShufflingReplicaListTransformer(r);
private RequestReplicaListTransformerGenerator requestReplicaListTransformerGenerator = new RequestReplicaListTransformerGenerator();
// URL scheme to be used in distributed search.
static final String INIT_URL_SCHEME = "urlScheme";
@ -224,6 +221,8 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
private void initReplicaListTransformers(NamedList routingConfig) {
String defaultRouting = null;
ReplicaListTransformerFactory stableRltFactory = null;
ReplicaListTransformerFactory defaultRltFactory;
if (routingConfig != null && routingConfig.size() > 0) {
Iterator<Entry<String,?>> iter = routingConfig.iterator();
do {
@ -240,21 +239,22 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
case ShardParams.REPLICA_STABLE:
NamedList<?> c = getNamedList(e.getValue());
defaultRouting = checkDefaultReplicaListTransformer(c, key, defaultRouting);
this.stableRltFactory = new AffinityReplicaListTransformerFactory(c);
stableRltFactory = new AffinityReplicaListTransformerFactory(c);
break;
default:
throw new IllegalArgumentException("invalid replica routing spec name: " + key);
}
} while (iter.hasNext());
}
if (this.stableRltFactory == null) {
this.stableRltFactory = new AffinityReplicaListTransformerFactory();
if (stableRltFactory == null) {
stableRltFactory = new AffinityReplicaListTransformerFactory();
}
if (ShardParams.REPLICA_STABLE.equals(defaultRouting)) {
this.defaultRltFactory = this.stableRltFactory;
defaultRltFactory = stableRltFactory;
} else {
this.defaultRltFactory = this.randomRltFactory;
defaultRltFactory = RequestReplicaListTransformerGenerator.RANDOM_RLTF;
}
this.requestReplicaListTransformerGenerator = new RequestReplicaListTransformerGenerator(defaultRltFactory, stableRltFactory);
}
@Override
@ -269,7 +269,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
String strategy = getParameter(args, "metricNameStrategy", UpdateShardHandlerConfig.DEFAULT_METRICNAMESTRATEGY, sb);
this.metricNameStrategy = KNOWN_METRIC_NAME_STRATEGIES.get(strategy);
if (this.metricNameStrategy == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
throw new SolrException(ErrorCode.SERVER_ERROR,
"Unknown metricNameStrategy: " + strategy + " found. Must be one of: " + KNOWN_METRIC_NAME_STRATEGIES.keySet());
}
@ -290,7 +290,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy,sb);
this.whitelistHostChecker = new WhitelistHostChecker(args == null? null: (String) args.get(INIT_SHARDS_WHITELIST), !getDisableShardsWhitelist());
log.info("Host whitelist initialized: {}", this.whitelistHostChecker);
// magic sysprop to make tests reproducible: set by SolrTestCaseJ4.
String v = System.getProperty("tests.shardhandler.randomSeed");
if (v != null) {
@ -354,7 +354,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
if (loadbalancer != null) {
loadbalancer.close();
}
} finally {
} finally {
if (defaultClient != null) {
IOUtils.closeQuietly(defaultClient);
}
@ -409,280 +409,23 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
return urls;
}
/**
* A distributed request is made via {@link LBSolrClient} to the first live server in the URL list.
* This means it is just as likely to choose current host as any of the other hosts.
* This function makes sure that the cores are sorted according to the given list of preferences.
* E.g. If all nodes prefer local cores then a bad/heavily-loaded node will receive less requests from
* healthy nodes. This will help prevent a distributed deadlock or timeouts in all the healthy nodes due
* to one bad node.
*
* Optional final preferenceRule is *not* used for pairwise sorting, but instead defines how "equivalent"
* replicas will be ordered (the base ordering). Defaults to "random"; may specify "stable".
*/
static class NodePreferenceRulesComparator implements Comparator<Object> {
private final SolrQueryRequest request;
private final NodesSysPropsCacher sysPropsCache;
private final String nodeName;
private final List<PreferenceRule> sortRules;
private final List<PreferenceRule> preferenceRules;
private String localHostAddress = null;
private final ReplicaListTransformer baseReplicaListTransformer;
public NodePreferenceRulesComparator(final List<PreferenceRule> preferenceRules, final SolrQueryRequest request,
final ReplicaListTransformerFactory defaultRltFactory, final ReplicaListTransformerFactory randomRltFactory,
final ReplicaListTransformerFactory stableRltFactory) {
this.request = request;
final SolrCore core; // explicit check for null core (temporary?, for tests)
if (request != null && (core = request.getCore()) != null && core.getCoreContainer().getZkController() != null) {
ZkController zkController = request.getCore().getCoreContainer().getZkController();
sysPropsCache = zkController.getSysPropsCacher();
nodeName = zkController.getNodeName();
} else {
sysPropsCache = null;
nodeName = null;
}
this.preferenceRules = preferenceRules;
final int maxIdx = preferenceRules.size() - 1;
final PreferenceRule lastRule = preferenceRules.get(maxIdx);
if (!ShardParams.SHARDS_PREFERENCE_REPLICA_BASE.equals(lastRule.name)) {
this.sortRules = preferenceRules;
this.baseReplicaListTransformer = defaultRltFactory.getInstance(null, request, randomRltFactory);
} else {
if (maxIdx == 0) {
this.sortRules = null;
} else {
this.sortRules = preferenceRules.subList(0, maxIdx);
}
String[] parts = lastRule.value.split(":", 2);
switch (parts[0]) {
case ShardParams.REPLICA_RANDOM:
this.baseReplicaListTransformer = randomRltFactory.getInstance(parts.length == 1 ? null : parts[1], request, null);
break;
case ShardParams.REPLICA_STABLE:
this.baseReplicaListTransformer = stableRltFactory.getInstance(parts.length == 1 ? null : parts[1], request, randomRltFactory);
break;
default:
throw new IllegalArgumentException("Invalid base replica order spec");
}
}
}
private static final ReplicaListTransformer NOOP_RLT = (List<?> choices) -> { /* noop */ };
private static final ReplicaListTransformerFactory NOOP_RLTF = (String configSpec, SolrQueryRequest request,
ReplicaListTransformerFactory fallback) -> NOOP_RLT;
/**
* For compatibility with tests, which expect this constructor to have no effect on the *base* order.
*/
NodePreferenceRulesComparator(final List<PreferenceRule> sortRules, final SolrQueryRequest request) {
this(sortRules, request, NOOP_RLTF, null, null);
}
public ReplicaListTransformer getBaseReplicaListTransformer() {
return baseReplicaListTransformer;
}
@Override
public int compare(Object left, Object right) {
if (this.sortRules != null) {
for (PreferenceRule preferenceRule: this.sortRules) {
final boolean lhs;
final boolean rhs;
switch (preferenceRule.name) {
case ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE:
lhs = hasReplicaType(left, preferenceRule.value);
rhs = hasReplicaType(right, preferenceRule.value);
break;
case ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION:
lhs = hasCoreUrlPrefix(left, preferenceRule.value);
rhs = hasCoreUrlPrefix(right, preferenceRule.value);
break;
case ShardParams.SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP:
if (sysPropsCache == null) {
throw new IllegalArgumentException("Unable to get the NodesSysPropsCacher" +
" on sorting replicas by preference:"+ preferenceRule.value);
}
lhs = hasSameMetric(left, preferenceRule.value);
rhs = hasSameMetric(right, preferenceRule.value);
break;
case ShardParams.SHARDS_PREFERENCE_REPLICA_BASE:
throw new IllegalArgumentException("only one base replica order may be specified in "
+ ShardParams.SHARDS_PREFERENCE + ", and it must be specified last");
default:
throw new IllegalArgumentException("Invalid " + ShardParams.SHARDS_PREFERENCE + " type: " + preferenceRule.name);
}
if (lhs != rhs) {
return lhs ? -1 : +1;
}
}
}
return 0;
}
private boolean hasSameMetric(Object o, String metricTag) {
if (!(o instanceof Replica)) {
return false;
}
Collection<String> tags = Collections.singletonList(metricTag);
String otherNodeName = ((Replica) o).getNodeName();
Map<String, Object> currentNodeMetric = sysPropsCache.getSysProps(nodeName, tags);
Map<String, Object> otherNodeMetric = sysPropsCache.getSysProps(otherNodeName, tags);
return currentNodeMetric.equals(otherNodeMetric);
}
private boolean hasCoreUrlPrefix(Object o, String prefix) {
final String s;
if (o instanceof String) {
s = (String)o;
}
else if (o instanceof Replica) {
s = ((Replica)o).getCoreUrl();
} else {
return false;
}
if (prefix.equals(ShardParams.REPLICA_LOCAL)) {
if (null == localHostAddress) {
final ZkController zkController = this.request.getCore().getCoreContainer().getZkController();
localHostAddress = zkController != null ? zkController.getBaseUrl() : "";
if (localHostAddress.isEmpty()) {
log.warn("Couldn't determine current host address for sorting of local replicas");
}
}
if (!localHostAddress.isEmpty()) {
if (s.startsWith(localHostAddress)) {
return true;
}
}
} else {
if (s.startsWith(prefix)) {
return true;
}
}
return false;
}
private static boolean hasReplicaType(Object o, String preferred) {
if (!(o instanceof Replica)) {
return false;
}
final String s = ((Replica)o).getType().toString();
return s.equals(preferred);
}
}
private final ReplicaListTransformerFactory randomRltFactory = (String configSpec, SolrQueryRequest request,
ReplicaListTransformerFactory fallback) -> shufflingReplicaListTransformer;
private ReplicaListTransformerFactory stableRltFactory;
private ReplicaListTransformerFactory defaultRltFactory;
/**
* Private class responsible for applying pairwise sort based on inherent replica attributes,
* and subsequently reordering any equivalent replica sets according to behavior specified
* by the baseReplicaListTransformer.
*/
private static final class TopLevelReplicaListTransformer implements ReplicaListTransformer {
private final NodePreferenceRulesComparator replicaComp;
private final ReplicaListTransformer baseReplicaListTransformer;
public TopLevelReplicaListTransformer(NodePreferenceRulesComparator replicaComp, ReplicaListTransformer baseReplicaListTransformer) {
this.replicaComp = replicaComp;
this.baseReplicaListTransformer = baseReplicaListTransformer;
}
@Override
public void transform(List<?> choices) {
if (choices.size() > 1) {
if (log.isDebugEnabled()) {
log.debug("Applying the following sorting preferences to replicas: {}",
Arrays.toString(replicaComp.preferenceRules.toArray()));
}
// First, sort according to comparator rules.
try {
choices.sort(replicaComp);
} catch (IllegalArgumentException iae) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
iae.getMessage()
);
}
// Next determine all boundaries between replicas ranked as "equivalent" by the comparator
Iterator<?> iter = choices.iterator();
Object prev = iter.next();
Object current;
int idx = 1;
int boundaryCount = 0;
int[] boundaries = new int[choices.size() - 1];
do {
current = iter.next();
if (replicaComp.compare(prev, current) != 0) {
boundaries[boundaryCount++] = idx;
}
prev = current;
idx++;
} while (iter.hasNext());
// Finally inspect boundaries to apply base transformation, where necessary (separate phase to avoid ConcurrentModificationException)
int startIdx = 0;
int endIdx;
for (int i = 0; i < boundaryCount; i++) {
endIdx = boundaries[i];
if (endIdx - startIdx > 1) {
baseReplicaListTransformer.transform(choices.subList(startIdx, endIdx));
}
startIdx = endIdx;
}
if (log.isDebugEnabled()) {
log.debug("Applied sorting preferences to replica list: {}",
Arrays.toString(choices.toArray()));
}
}
}
}
protected ReplicaListTransformer getReplicaListTransformer(final SolrQueryRequest req) {
final SolrParams params = req.getParams();
final SolrCore core = req.getCore(); // explicit check for null core (temporary?, for tests)
ZkController zkController = core == null ? null : core.getCoreContainer().getZkController();
String defaultShardPreference = "";
if (zkController != null) {
defaultShardPreference = zkController.getZkStateReader().getClusterProperties()
.getOrDefault(ZkStateReader.DEFAULT_SHARD_PREFERENCES, "")
.toString();
return requestReplicaListTransformerGenerator.getReplicaListTransformer(
params,
zkController.getZkStateReader().getClusterProperties()
.getOrDefault(ZkStateReader.DEFAULT_SHARD_PREFERENCES, "")
.toString(),
zkController.getNodeName(),
zkController.getBaseUrl(),
zkController.getSysPropsCacher()
);
} else {
return requestReplicaListTransformerGenerator.getReplicaListTransformer(params);
}
@SuppressWarnings("deprecation")
final boolean preferLocalShards = params.getBool(CommonParams.PREFER_LOCAL_SHARDS, false);
final String shardsPreferenceSpec = params.get(ShardParams.SHARDS_PREFERENCE, defaultShardPreference);
if (preferLocalShards || !shardsPreferenceSpec.isEmpty()) {
if (preferLocalShards && !shardsPreferenceSpec.isEmpty()) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"preferLocalShards is deprecated and must not be used with shards.preference"
);
}
List<PreferenceRule> preferenceRules = PreferenceRule.from(shardsPreferenceSpec);
if (preferLocalShards) {
preferenceRules.add(new PreferenceRule(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION, ShardParams.REPLICA_LOCAL));
}
NodePreferenceRulesComparator replicaComp = new NodePreferenceRulesComparator(preferenceRules, req,
defaultRltFactory, randomRltFactory, stableRltFactory);
ReplicaListTransformer baseReplicaListTransformer = replicaComp.getBaseReplicaListTransformer();
if (replicaComp.sortRules == null) {
// only applying base transformation
return baseReplicaListTransformer;
} else {
return new TopLevelReplicaListTransformer(replicaComp, baseReplicaListTransformer);
}
}
return defaultRltFactory.getInstance(null, req, randomRltFactory);
}
/**
@ -691,7 +434,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
public CompletionService newCompletionService() {
return new ExecutorCompletionService<ShardResponse>(commExecutor);
}
/**
* Rebuilds the URL replacing the URL scheme of the passed URL with the
* configured scheme replacement.If no scheme was configured, the passed URL's
@ -703,7 +446,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
} else if(StringUtils.isNotEmpty(scheme)) {
return scheme + "://" + URLUtil.removeScheme(url);
}
return url;
}
@ -716,28 +459,28 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
solrMetricsContext.getMetricRegistry(),
SolrMetricManager.mkName("httpShardExecutor", expandedScope, "threadPool"));
}
/**
* Class used to validate the hosts in the "shards" parameter when doing a distributed
* request
*/
public static class WhitelistHostChecker {
/**
* List of the whitelisted hosts. Elements in the list will be host:port (no protocol or context)
*/
private final Set<String> whitelistHosts;
/**
* Indicates whether host checking is enabled
* Indicates whether host checking is enabled
*/
private final boolean whitelistHostCheckingEnabled;
public WhitelistHostChecker(String whitelistStr, boolean enabled) {
this.whitelistHosts = implGetShardsWhitelist(whitelistStr);
this.whitelistHostCheckingEnabled = enabled;
}
final static Set<String> implGetShardsWhitelist(final String shardsWhitelist) {
if (shardsWhitelist != null && !shardsWhitelist.isEmpty()) {
return StrUtils.splitSmart(shardsWhitelist, ',')
@ -753,32 +496,32 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
url = new URL(hostUrl);
}
} catch (MalformedURLException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist, e);
throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist, e);
}
if (url.getHost() == null || url.getPort() < 0) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist);
throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist);
}
return url.getHost() + ":" + url.getPort();
}).collect(Collectors.toSet());
}
return null;
}
/**
* @see #checkWhitelist(ClusterState, String, List)
*/
protected void checkWhitelist(String shardsParamValue, List<String> shardUrls) {
checkWhitelist(null, shardsParamValue, shardUrls);
}
/**
* Checks that all the hosts for all the shards requested in shards parameter exist in the configured whitelist
* or in the ClusterState (in case of cloud mode)
*
*
* @param clusterState The up to date ClusterState, can be null in case of non-cloud mode
* @param shardsParamValue The original shards parameter
* @param shardUrls The list of cores generated from the shards parameter.
* @param shardUrls The list of cores generated from the shards parameter.
*/
protected void checkWhitelist(ClusterState clusterState, String shardsParamValue, List<String> shardUrls) {
if (!whitelistHostCheckingEnabled) {
@ -793,7 +536,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
} else {
localWhitelistHosts = Collections.emptySet();
}
shardUrls.stream().map(String::trim).forEach((shardUrl) -> {
URL url;
try {
@ -804,10 +547,10 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
url = new URL(shardUrl);
}
} catch (MalformedURLException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue, e);
throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue, e);
}
if (url.getHost() == null || url.getPort() < 0) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue);
throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue);
}
if (!localWhitelistHosts.contains(url.getHost() + ":" + url.getPort())) {
log.warn("The '"+ShardParams.SHARDS+"' parameter value '"+shardsParamValue+"' contained value(s) not on the shards whitelist ("+localWhitelistHosts+"), shardUrl:" + shardUrl);

8
solr/core/src/java/org/apache/solr/util/TestInjection.java
View File

@ -20,7 +20,6 @@ import java.lang.invoke.MethodHandles;
import java.lang.reflect.Method;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.Timer;
@ -147,8 +146,6 @@ public class TestInjection {
public volatile static boolean uifOutOfMemoryError = false;
public volatile static Map<String, String> additionalSystemProps = null;
private volatile static CountDownLatch notifyPauseForeverDone = new CountDownLatch(1);
public static void notifyPauseForeverDone() {
@ -157,7 +154,6 @@ public class TestInjection {
}
public static void reset() {
additionalSystemProps = null;
nonGracefullClose = null;
failReplicaRequests = null;
failUpdateRequests = null;
@ -494,10 +490,6 @@ public class TestInjection {
return true;
}
public static Map<String,String> injectAdditionalProps() {
return additionalSystemProps;
}
public static boolean injectUIFOutOfMemoryError() {
if (uifOutOfMemoryError ) {
throw new OutOfMemoryError("Test Injection");

9
solr/core/src/test/org/apache/solr/cloud/RoutingToNodesWithPropertiesTest.java
View File

@ -41,6 +41,7 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.CommonTestInjection;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.TimeSource;
@ -67,14 +68,14 @@ public class RoutingToNodesWithPropertiesTest extends SolrCloudTestCase {
@Before
public void setupCluster() throws Exception {
TestInjection.additionalSystemProps = ImmutableMap.of("zone", "us-west1");
CommonTestInjection.setAdditionalProps(ImmutableMap.of("zone", "us-west1"));
configureCluster(2)
.withSolrXml(TEST_PATH().resolve("solr-trackingshardhandler.xml"))
.addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
zone1Nodes.addAll(cluster.getJettySolrRunners().stream().map(JettySolrRunner::getNodeName).collect(Collectors.toSet()));
TestInjection.additionalSystemProps = ImmutableMap.of("zone", "us-west2");
CommonTestInjection.setAdditionalProps(ImmutableMap.of("zone", "us-west2"));
zone2Nodes.add(cluster.startJettySolrRunner().getNodeName());
zone2Nodes.add(cluster.startJettySolrRunner().getNodeName());
@ -114,9 +115,9 @@ public class RoutingToNodesWithPropertiesTest extends SolrCloudTestCase {
assertEquals("us-west1", map.get(PROP_NAME));
}
for (String zone1Node: zone2Nodes) {
for (String zone2Node: zone2Nodes) {
NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
Map<String, Object> map = nodeStateProvider.getNodeValues(zone1Node, Collections.singletonList(PROP_NAME));
Map<String, Object> map = nodeStateProvider.getNodeValues(zone2Node, Collections.singletonList(PROP_NAME));
assertEquals("us-west2", map.get(PROP_NAME));
}

211
solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
View File

@ -20,7 +20,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -28,19 +27,11 @@ import java.util.Set;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.LBSolrClient;
import org.apache.solr.client.solrj.impl.PreferenceRule;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.HttpShardHandlerFactory.WhitelistHostChecker;
import org.apache.solr.request.SolrQueryRequestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -121,208 +112,6 @@ public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 {
}
}
@SuppressWarnings("unchecked")
public void testNodePreferenceRulesBase() throws Exception {
SolrCore testCore = null;
HttpShardHandlerFactory fac = new HttpShardHandlerFactory();
fac.init(new PluginInfo(null, Collections.EMPTY_MAP));
SolrQueryRequestBase req;
NamedList<String> params = new NamedList<>();
List<Replica> replicas = getBasicReplicaList();
String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
params.add("routingPreference", "0");
params.add(ShardParams.SHARDS_PREFERENCE, rulesParam);
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
ReplicaListTransformer rlt = fac.getReplicaListTransformer(req);
rlt.transform(replicas);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node2", replicas.get(1).getNodeName());
assertEquals("node3", replicas.get(2).getNodeName());
req.close();
params.setVal(0, "1");
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
rlt = fac.getReplicaListTransformer(req);
rlt.transform(replicas);
assertEquals("node2", replicas.get(0).getNodeName());
assertEquals("node3", replicas.get(1).getNodeName());
assertEquals("node1", replicas.get(2).getNodeName());
req.close();
params.setVal(0, "2");
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
rlt = fac.getReplicaListTransformer(req);
rlt.transform(replicas);
assertEquals("node3", replicas.get(0).getNodeName());
assertEquals("node1", replicas.get(1).getNodeName());
assertEquals("node2", replicas.get(2).getNodeName());
req.close();
params.setVal(0, "3");
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
rlt = fac.getReplicaListTransformer(req);
rlt.transform(replicas);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node2", replicas.get(1).getNodeName());
assertEquals("node3", replicas.get(2).getNodeName());
req.close();
// Add a replica so that sorting by replicaType:TLOG can cause a tie
replicas.add(
new Replica(
"node4",
map(
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node4",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "TLOG"
)
)
);
// replicaType and replicaBase combined rule param
rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
params.setVal(0, "0");
params.setVal(1, rulesParam);
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
rlt = fac.getReplicaListTransformer(req);
rlt.transform(replicas);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node2", replicas.get(1).getNodeName());
assertEquals("node4", replicas.get(2).getNodeName());
assertEquals("node3", replicas.get(3).getNodeName());
req.close();
params.setVal(0, "1");
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
rlt = fac.getReplicaListTransformer(req);
rlt.transform(replicas);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node4", replicas.get(1).getNodeName());
assertEquals("node2", replicas.get(2).getNodeName());
assertEquals("node3", replicas.get(3).getNodeName());
req.close();
fac.close();
}
@SuppressWarnings("unchecked")
private static List<Replica> getBasicReplicaList() {
List<Replica> replicas = new ArrayList<Replica>();
replicas.add(
new Replica(
"node1",
map(
ZkStateReader.BASE_URL_PROP, "http://host1:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node1",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "NRT"
)
)
);
replicas.add(
new Replica(
"node2",
map(
ZkStateReader.BASE_URL_PROP, "http://host2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node2",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "TLOG"
)
)
);
replicas.add(
new Replica(
"node3",
map(
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node3",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "PULL"
)
)
);
return replicas;
}
@SuppressWarnings("unchecked")
public void testNodePreferenceRulesComparator() throws Exception {
List<Replica> replicas = getBasicReplicaList();
// Simple replica type rule
List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG");
HttpShardHandlerFactory.NodePreferenceRulesComparator comparator =
new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node2", replicas.get(1).getNodeName());
// Another simple replica type rule
rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT");
comparator = new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
assertEquals("node2", replicas.get(0).getNodeName());
assertEquals("node1", replicas.get(1).getNodeName());
// replicaLocation rule
rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2:8983");
comparator = new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
assertEquals("node2", replicas.get(0).getNodeName());
assertEquals("node1", replicas.get(1).getNodeName());
// Add a replica so that sorting by replicaType:TLOG can cause a tie
replicas.add(
new Replica(
"node4",
map(
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node4",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "TLOG"
)
)
);
// replicaType and replicaLocation combined rule
rules = PreferenceRule.from(
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2_2");
comparator = new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node4", replicas.get(1).getNodeName());
assertEquals("node2", replicas.get(2).getNodeName());
assertEquals("node3", replicas.get(3).getNodeName());
// Bad rule
try {
rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE);
fail();
} catch (IllegalArgumentException e) {
assertEquals("Invalid shards.preference rule: " + ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE, e.getMessage());
}
// Unknown rule
rules = PreferenceRule.from("badRule:test");
try {
comparator = new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
fail();
} catch (IllegalArgumentException e) {
assertEquals("Invalid shards.preference type: badRule", e.getMessage());
}
}
@Test
public void getShardsWhitelist() throws Exception {
System.setProperty(SHARDS_WHITELIST, "http://abc:8983/,http://def:8984/,");

14
solr/core/src/java/org/apache/solr/handler/component/AffinityReplicaListTransformer.java → solr/solrj/src/java/org/apache/solr/client/solrj/routing/AffinityReplicaListTransformer.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
package org.apache.solr.client.solrj.routing;
import java.util.Arrays;
import java.util.Comparator;
@ -24,14 +24,13 @@ import java.util.ListIterator;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.Hash;
import org.apache.solr.request.SolrQueryRequest;
/**
* Allows better caching by establishing deterministic evenly-distributed replica routing preferences according to
* either explicitly configured hash routing parameter, or the hash of a query parameter (configurable, usually related
* to the main query).
*/
class AffinityReplicaListTransformer implements ReplicaListTransformer {
public class AffinityReplicaListTransformer implements ReplicaListTransformer {
private final int routingDividend;
@ -47,17 +46,16 @@ class AffinityReplicaListTransformer implements ReplicaListTransformer {
*
* @param dividendParam int param to be used directly for mod-based routing
* @param hashParam String param to be hashed into an int for mod-based routing
* @param req the request from which param values will be drawn
* @param requestParams the parameters of the Solr request
* @return null if specified routing vals are not able to be parsed properly
*/
public static ReplicaListTransformer getInstance(String dividendParam, String hashParam, SolrQueryRequest req) {
SolrParams params = req.getOriginalParams();
public static ReplicaListTransformer getInstance(String dividendParam, String hashParam, SolrParams requestParams) {
Integer dividendVal;
if (dividendParam != null && (dividendVal = params.getInt(dividendParam)) != null) {
if (dividendParam != null && (dividendVal = requestParams.getInt(dividendParam)) != null) {
return new AffinityReplicaListTransformer(dividendVal);
}
String hashVal;
if (hashParam != null && (hashVal = params.get(hashParam)) != null && !hashVal.isEmpty()) {
if (hashParam != null && (hashVal = requestParams.get(hashParam)) != null && !hashVal.isEmpty()) {
return new AffinityReplicaListTransformer(hashVal);
} else {
return null;

18
solr/core/src/java/org/apache/solr/handler/component/AffinityReplicaListTransformerFactory.java → solr/solrj/src/java/org/apache/solr/client/solrj/routing/AffinityReplicaListTransformerFactory.java
View File

@ -14,12 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
package org.apache.solr.client.solrj.routing;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
/**
* Factory for constructing an {@link AffinityReplicaListTransformer} that reorders replica routing
@ -27,12 +27,12 @@ import org.apache.solr.request.SolrQueryRequest;
*
* Default names of params that contain the values by which routing is determined may be configured
* at the time of {@link AffinityReplicaListTransformerFactory} construction, and may be
* overridden by the config spec passed to {@link #getInstance(String, SolrQueryRequest, ReplicaListTransformerFactory)}
* overridden by the config spec passed to {@link #getInstance(String, SolrParams, ReplicaListTransformerFactory)}
*
* If no defaultHashParam name is specified at time of factory construction, the routing dividend will
* be derived by hashing the {@link String} value of the {@link CommonParams#Q} param.
*/
class AffinityReplicaListTransformerFactory implements ReplicaListTransformerFactory {
public class AffinityReplicaListTransformerFactory implements ReplicaListTransformerFactory {
private final String defaultDividendParam;
private final String defaultHashParam;
@ -68,24 +68,24 @@ class AffinityReplicaListTransformerFactory implements ReplicaListTransformerFac
}
@Override
public ReplicaListTransformer getInstance(String configSpec, SolrQueryRequest request, ReplicaListTransformerFactory fallback) {
public ReplicaListTransformer getInstance(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback) {
ReplicaListTransformer rlt;
if (configSpec == null) {
rlt = AffinityReplicaListTransformer.getInstance(defaultDividendParam, defaultHashParam, request);
rlt = AffinityReplicaListTransformer.getInstance(defaultDividendParam, defaultHashParam, requestParams);
} else {
String[] parts = configSpec.split(":", 2);
switch (parts[0]) {
case ShardParams.ROUTING_DIVIDEND:
rlt = AffinityReplicaListTransformer.getInstance(parts.length == 1 ? defaultDividendParam : parts[1], defaultHashParam, request);
rlt = AffinityReplicaListTransformer.getInstance(parts.length == 1 ? defaultDividendParam : parts[1], defaultHashParam, requestParams);
break;
case ShardParams.ROUTING_HASH:
rlt = AffinityReplicaListTransformer.getInstance(null, parts.length == 1 ? defaultHashParam : parts[1], request);
rlt = AffinityReplicaListTransformer.getInstance(null, parts.length == 1 ? defaultHashParam : parts[1], requestParams);
break;
default:
throw new IllegalArgumentException("Invalid routing spec: \"" + configSpec + '"');
}
}
return rlt != null ? rlt : fallback.getInstance(null, request, null);
return rlt != null ? rlt : fallback.getInstance(null, requestParams, null);
}
}

179
solr/solrj/src/java/org/apache/solr/client/solrj/routing/NodePreferenceRulesComparator.java Normal file
View File

@ -0,0 +1,179 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.routing;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.StringUtils;
import org.apache.solr.common.cloud.NodesSysPropsCacher;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
/**
* This comparator makes sure that the given replicas are sorted according to the given list of preferences.
* E.g. If all nodes prefer local cores then a bad/heavily-loaded node will receive less requests from
* healthy nodes. This will help prevent a distributed deadlock or timeouts in all the healthy nodes due
* to one bad node.
*
* Optional final preferenceRule is *not* used for pairwise sorting, but instead defines how "equivalent"
* replicas will be ordered (the base ordering). Defaults to "random"; may specify "stable".
*/
public class NodePreferenceRulesComparator implements Comparator<Object> {
private final NodesSysPropsCacher sysPropsCache;
private final String nodeName;
private final List<PreferenceRule> sortRules;
private final List<PreferenceRule> preferenceRules;
private final String localHostAddress;
private final ReplicaListTransformer baseReplicaListTransformer;
public NodePreferenceRulesComparator(final List<PreferenceRule> preferenceRules, final SolrParams requestParams,
final ReplicaListTransformerFactory defaultRltFactory, final ReplicaListTransformerFactory stableRltFactory) {
this(preferenceRules, requestParams, null, null, null, defaultRltFactory, stableRltFactory);
}
public NodePreferenceRulesComparator(final List<PreferenceRule> preferenceRules, final SolrParams requestParams,
final String nodeName, final String localHostAddress, final NodesSysPropsCacher sysPropsCache,
final ReplicaListTransformerFactory defaultRltFactory, final ReplicaListTransformerFactory stableRltFactory) {
this.sysPropsCache = sysPropsCache;
this.preferenceRules = preferenceRules;
this.nodeName = nodeName;
this.localHostAddress = localHostAddress;
final int maxIdx = preferenceRules.size() - 1;
final PreferenceRule lastRule = preferenceRules.get(maxIdx);
if (!ShardParams.SHARDS_PREFERENCE_REPLICA_BASE.equals(lastRule.name)) {
this.sortRules = preferenceRules;
this.baseReplicaListTransformer = defaultRltFactory.getInstance(null, requestParams, RequestReplicaListTransformerGenerator.RANDOM_RLTF);
} else {
if (maxIdx == 0) {
this.sortRules = null;
} else {
this.sortRules = preferenceRules.subList(0, maxIdx);
}
String[] parts = lastRule.value.split(":", 2);
switch (parts[0]) {
case ShardParams.REPLICA_RANDOM:
this.baseReplicaListTransformer = RequestReplicaListTransformerGenerator.RANDOM_RLTF.getInstance(parts.length == 1 ? null : parts[1], requestParams, null);
break;
case ShardParams.REPLICA_STABLE:
this.baseReplicaListTransformer = stableRltFactory.getInstance(parts.length == 1 ? null : parts[1], requestParams, RequestReplicaListTransformerGenerator.RANDOM_RLTF);
break;
default:
throw new IllegalArgumentException("Invalid base replica order spec");
}
}
}
private static final ReplicaListTransformer NOOP_RLT = (List<?> choices) -> { /* noop */ };
private static final ReplicaListTransformerFactory NOOP_RLTF =
(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback) -> NOOP_RLT;
/**
* For compatibility with tests, which expect this constructor to have no effect on the *base* order.
*/
NodePreferenceRulesComparator(final List<PreferenceRule> sortRules, final SolrParams requestParams) {
this(sortRules, requestParams, NOOP_RLTF, null);
}
public ReplicaListTransformer getBaseReplicaListTransformer() {
return baseReplicaListTransformer;
}
@Override
public int compare(Object left, Object right) {
if (this.sortRules != null) {
for (PreferenceRule preferenceRule: this.sortRules) {
final boolean lhs;
final boolean rhs;
switch (preferenceRule.name) {
case ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE:
lhs = hasReplicaType(left, preferenceRule.value);
rhs = hasReplicaType(right, preferenceRule.value);
break;
case ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION:
lhs = hasCoreUrlPrefix(left, preferenceRule.value);
rhs = hasCoreUrlPrefix(right, preferenceRule.value);
break;
case ShardParams.SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP:
if (sysPropsCache == null) {
throw new IllegalArgumentException("Unable to get the NodesSysPropsCacher on sorting replicas by preference:"+ preferenceRule.value);
}
lhs = hasSameMetric(left, preferenceRule.value);
rhs = hasSameMetric(right, preferenceRule.value);
break;
case ShardParams.SHARDS_PREFERENCE_REPLICA_BASE:
throw new IllegalArgumentException("only one base replica order may be specified in "
+ ShardParams.SHARDS_PREFERENCE + ", and it must be specified last");
default:
throw new IllegalArgumentException("Invalid " + ShardParams.SHARDS_PREFERENCE + " type: " + preferenceRule.name);
}
if (lhs != rhs) {
return lhs ? -1 : +1;
}
}
}
return 0;
}
private boolean hasSameMetric(Object o, String metricTag) {
if (!(o instanceof Replica)) {
return false;
}
Collection<String> tags = Collections.singletonList(metricTag);
String otherNodeName = ((Replica) o).getNodeName();
Map<String, Object> currentNodeMetric = sysPropsCache.getSysProps(nodeName, tags);
Map<String, Object> otherNodeMetric = sysPropsCache.getSysProps(otherNodeName, tags);
return currentNodeMetric.equals(otherNodeMetric);
}
private boolean hasCoreUrlPrefix(Object o, String prefix) {
final String s;
if (o instanceof String) {
s = (String)o;
}
else if (o instanceof Replica) {
s = ((Replica)o).getCoreUrl();
} else {
return false;
}
if (prefix.equals(ShardParams.REPLICA_LOCAL)) {
return !StringUtils.isEmpty(localHostAddress) && s.startsWith(localHostAddress);
} else {
return s.startsWith(prefix);
}
}
private static boolean hasReplicaType(Object o, String preferred) {
if (!(o instanceof Replica)) {
return false;
}
final String s = ((Replica)o).getType().toString();
return s.equals(preferred);
}
public List<PreferenceRule> getSortRules() {
return sortRules;
}
public List<PreferenceRule> getPreferenceRules() {
return preferenceRules;
}
}

2
solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreferenceRule.java → solr/solrj/src/java/org/apache/solr/client/solrj/routing/PreferenceRule.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
package org.apache.solr.client.solrj.routing;
import java.util.ArrayList;
import java.util.List;

2
solr/core/src/java/org/apache/solr/handler/component/ReplicaListTransformer.java → solr/solrj/src/java/org/apache/solr/client/solrj/routing/ReplicaListTransformer.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
package org.apache.solr.client.solrj.routing;
import java.util.List;

8
solr/core/src/java/org/apache/solr/handler/component/ReplicaListTransformerFactory.java → solr/solrj/src/java/org/apache/solr/client/solrj/routing/ReplicaListTransformerFactory.java
View File

@ -14,21 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
package org.apache.solr.client.solrj.routing;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.common.params.SolrParams;
public interface ReplicaListTransformerFactory {
/**
*
* @param configSpec spec for dynamic configuration of ReplicaListTransformer
* @param request the request for which the ReplicaListTransformer is being generated
* @param requestParams the request parameters for which the ReplicaListTransformer is being generated
* @param fallback used to generate fallback value; the getInstance() method of the specified fallback must not
* return null; The fallback value itself may be null if this implementation is known to never return null (i.e., if
* fallback will never be needed)
* @return ReplicaListTransformer to be used for routing this request
*/
ReplicaListTransformer getInstance(String configSpec, SolrQueryRequest request, ReplicaListTransformerFactory fallback);
ReplicaListTransformer getInstance(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback);
}

168
solr/solrj/src/java/org/apache/solr/client/solrj/routing/RequestReplicaListTransformerGenerator.java Normal file
View File

@ -0,0 +1,168 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.routing;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.NodesSysPropsCacher;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RequestReplicaListTransformerGenerator {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final Random r = new Random();
private static final ReplicaListTransformer shufflingReplicaListTransformer = new ShufflingReplicaListTransformer(r);
public static final ReplicaListTransformerFactory RANDOM_RLTF =
(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback) -> shufflingReplicaListTransformer;
private final ReplicaListTransformerFactory stableRltFactory;
private final ReplicaListTransformerFactory defaultRltFactory;
public RequestReplicaListTransformerGenerator() {
this(RANDOM_RLTF);
}
public RequestReplicaListTransformerGenerator(ReplicaListTransformerFactory defaultRltFactory) {
this(defaultRltFactory, null);
}
public RequestReplicaListTransformerGenerator(ReplicaListTransformerFactory defaultRltFactory, ReplicaListTransformerFactory stableRltFactory) {
this.defaultRltFactory = defaultRltFactory;
if (stableRltFactory == null) {
this.stableRltFactory = new AffinityReplicaListTransformerFactory();
} else {
this.stableRltFactory = stableRltFactory;
}
}
public ReplicaListTransformer getReplicaListTransformer(final SolrParams requestParams) {
return getReplicaListTransformer(requestParams, "");
}
public ReplicaListTransformer getReplicaListTransformer(final SolrParams requestParams, String defaultShardPreferences) {
return getReplicaListTransformer(requestParams, defaultShardPreferences, null, null, null);
}
public ReplicaListTransformer getReplicaListTransformer(final SolrParams requestParams, String defaultShardPreferences, String nodeName, String localHostAddress, NodesSysPropsCacher sysPropsCacher) {
@SuppressWarnings("deprecation")
final boolean preferLocalShards = requestParams.getBool(CommonParams.PREFER_LOCAL_SHARDS, false);
final String shardsPreferenceSpec = requestParams.get(ShardParams.SHARDS_PREFERENCE, defaultShardPreferences);
if (preferLocalShards || !shardsPreferenceSpec.isEmpty()) {
if (preferLocalShards && !shardsPreferenceSpec.isEmpty()) {
throw new SolrException(
ErrorCode.BAD_REQUEST,
"preferLocalShards is deprecated and must not be used with shards.preference"
);
}
List<PreferenceRule> preferenceRules = PreferenceRule.from(shardsPreferenceSpec);
if (preferLocalShards) {
preferenceRules.add(new PreferenceRule(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION, ShardParams.REPLICA_LOCAL));
}
NodePreferenceRulesComparator replicaComp = new NodePreferenceRulesComparator(preferenceRules, requestParams, nodeName, localHostAddress, sysPropsCacher, defaultRltFactory, stableRltFactory);
ReplicaListTransformer baseReplicaListTransformer = replicaComp.getBaseReplicaListTransformer();
if (replicaComp.getSortRules() == null) {
// only applying base transformation
return baseReplicaListTransformer;
} else {
return new TopLevelReplicaListTransformer(replicaComp, baseReplicaListTransformer);
}
}
return defaultRltFactory.getInstance(null, requestParams, RANDOM_RLTF);
}
/**
* Private class responsible for applying pairwise sort based on inherent replica attributes,
* and subsequently reordering any equivalent replica sets according to behavior specified
* by the baseReplicaListTransformer.
*/
private static final class TopLevelReplicaListTransformer implements ReplicaListTransformer {
private final NodePreferenceRulesComparator replicaComp;
private final ReplicaListTransformer baseReplicaListTransformer;
public TopLevelReplicaListTransformer(NodePreferenceRulesComparator replicaComp, ReplicaListTransformer baseReplicaListTransformer) {
this.replicaComp = replicaComp;
this.baseReplicaListTransformer = baseReplicaListTransformer;
}
@Override
public void transform(List<?> choices) {
if (choices.size() > 1) {
if (log.isDebugEnabled()) {
log.debug("Applying the following sorting preferences to replicas: {}",
Arrays.toString(replicaComp.getPreferenceRules().toArray()));
}
// First, sort according to comparator rules.
try {
choices.sort(replicaComp);
} catch (IllegalArgumentException iae) {
throw new SolrException(
ErrorCode.BAD_REQUEST,
iae.getMessage()
);
}
// Next determine all boundaries between replicas ranked as "equivalent" by the comparator
Iterator<?> iter = choices.iterator();
Object prev = iter.next();
Object current;
int idx = 1;
int boundaryCount = 0;
int[] boundaries = new int[choices.size() - 1];
do {
current = iter.next();
if (replicaComp.compare(prev, current) != 0) {
boundaries[boundaryCount++] = idx;
}
prev = current;
idx++;
} while (iter.hasNext());
// Finally inspect boundaries to apply base transformation, where necessary (separate phase to avoid ConcurrentModificationException)
int startIdx = 0;
int endIdx;
for (int i = 0; i < boundaryCount; i++) {
endIdx = boundaries[i];
if (endIdx - startIdx > 1) {
baseReplicaListTransformer.transform(choices.subList(startIdx, endIdx));
}
startIdx = endIdx;
}
if (log.isDebugEnabled()) {
log.debug("Applied sorting preferences to replica list: {}",
Arrays.toString(choices.toArray()));
}
}
}
}
}

4
solr/core/src/java/org/apache/solr/handler/component/ShufflingReplicaListTransformer.java → solr/solrj/src/java/org/apache/solr/client/solrj/routing/ShufflingReplicaListTransformer.java
View File

@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
package org.apache.solr.client.solrj.routing;
import java.util.Collections;
import java.util.List;
import java.util.Random;
class ShufflingReplicaListTransformer implements ReplicaListTransformer {
public class ShufflingReplicaListTransformer implements ReplicaListTransformer {
private final Random r;

18
solr/core/src/java/org/apache/solr/cloud/NodesSysPropsCacher.java → solr/solrj/src/java/org/apache/solr/common/cloud/NodesSysPropsCacher.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.solr.cloud;
package org.apache.solr.common.cloud;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
@ -29,13 +29,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.impl.PreferenceRule;
import org.apache.solr.client.solrj.routing.PreferenceRule;
import org.apache.solr.common.SolrCloseable;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.util.TestInjection;
import org.apache.solr.common.util.CommonTestInjection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -55,7 +53,7 @@ public class NodesSysPropsCacher implements SolrCloseable {
private final AtomicBoolean isRunning = new AtomicBoolean(false);
private final NodeStateProvider nodeStateProvider;
private final Map<String, String> additionalProps = TestInjection.injectAdditionalProps();
private Map<String, String> additionalProps = CommonTestInjection.injectAdditionalProps();
private final String currentNode;
private final ConcurrentHashMap<String, Map<String, Object>> cache = new ConcurrentHashMap<>();
private final AtomicInteger fetchCounting = new AtomicInteger(0);
@ -63,9 +61,9 @@ public class NodesSysPropsCacher implements SolrCloseable {
private volatile boolean isClosed;
private volatile Collection<String> tags = new ArrayList<>();
NodesSysPropsCacher(NodeStateProvider nodeStateProvider,
String currentNode,
ZkStateReader stateReader) {
public NodesSysPropsCacher(NodeStateProvider nodeStateProvider,
String currentNode,
ZkStateReader stateReader) {
this.nodeStateProvider = nodeStateProvider;
this.currentNode = currentNode;
@ -177,12 +175,10 @@ public class NodesSysPropsCacher implements SolrCloseable {
return result;
}
@VisibleForTesting
public int getCacheSize() {
return cache.size();
}
@VisibleForTesting
public boolean isRunning() {
return isRunning.get();
}

42
solr/solrj/src/java/org/apache/solr/common/util/CommonTestInjection.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.util;
import java.util.Map;
/**
* Allows random faults to be injected in running code during test runs across all solr packages.
*
* @lucene.internal
*/
public class CommonTestInjection {
private volatile static Map<String, String> additionalSystemProps = null;
public static void reset() {
additionalSystemProps = null;
}
public static void setAdditionalProps(Map<String, String> additionalSystemProps) {
CommonTestInjection.additionalSystemProps = additionalSystemProps;
}
public static Map<String,String> injectAdditionalProps() {
return additionalSystemProps;
}
}

155
solr/solrj/src/test/org/apache/solr/client/solrj/routing/NodePreferenceRulesComparatorTest.java Normal file
View File

@ -0,0 +1,155 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.routing;
import java.util.ArrayList;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ShardParams;
import org.junit.Test;
public class NodePreferenceRulesComparatorTest extends SolrTestCaseJ4 {
@Test
public void replicaLocationTest() {
List<Replica> replicas = getBasicReplicaList();
// replicaLocation rule
List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2:8983");
NodePreferenceRulesComparator comparator = new NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
assertEquals("node2", replicas.get(0).getNodeName());
assertEquals("node1", replicas.get(1).getNodeName());
}
public void replicaTypeTest() {
List<Replica> replicas = getBasicReplicaList();
List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG");
NodePreferenceRulesComparator comparator = new NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node2", replicas.get(1).getNodeName());
// reversed rule
rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT");
comparator = new NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
assertEquals("node2", replicas.get(0).getNodeName());
assertEquals("node1", replicas.get(1).getNodeName());
}
@SuppressWarnings("unchecked")
@Test
public void replicaTypeAndReplicaLocationTest() {
List<Replica> replicas = getBasicReplicaList();
// Add a replica so that sorting by replicaType:TLOG can cause a tie
replicas.add(
new Replica(
"node4",
map(
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node4",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "TLOG"
)
)
);
List<PreferenceRule> rules = PreferenceRule.from(
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2_2");
NodePreferenceRulesComparator comparator = new NodePreferenceRulesComparator(rules, null);
replicas.sort(comparator);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node4", replicas.get(1).getNodeName());
assertEquals("node2", replicas.get(2).getNodeName());
assertEquals("node3", replicas.get(3).getNodeName());
}
@Test(expected = IllegalArgumentException.class)
public void badRuleTest() {
try {
PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE);
} catch (IllegalArgumentException e) {
assertEquals("Invalid shards.preference rule: " + ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE, e.getMessage());
throw e;
}
}
@Test(expected = IllegalArgumentException.class)
public void unknownRuleTest() {
List<Replica> replicas = getBasicReplicaList();
List<PreferenceRule> rules = PreferenceRule.from("badRule:test");
try {
replicas.sort(new NodePreferenceRulesComparator(rules, null));
} catch (IllegalArgumentException e) {
assertEquals("Invalid shards.preference type: badRule", e.getMessage());
throw e;
}
}
@SuppressWarnings("unchecked")
private static List<Replica> getBasicReplicaList() {
List<Replica> replicas = new ArrayList<Replica>();
replicas.add(
new Replica(
"node1",
map(
ZkStateReader.BASE_URL_PROP, "http://host1:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node1",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "NRT"
)
)
);
replicas.add(
new Replica(
"node2",
map(
ZkStateReader.BASE_URL_PROP, "http://host2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node2",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "TLOG"
)
)
);
replicas.add(
new Replica(
"node3",
map(
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node3",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "PULL"
)
)
);
return replicas;
}
}

3
solr/core/src/test/org/apache/solr/handler/component/ReplicaListTransformerTest.java → solr/solrj/src/test/org/apache/solr/client/solrj/routing/ReplicaListTransformerTest.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
package org.apache.solr.client.solrj.routing;
import java.util.ArrayList;
import java.util.Collections;
@ -28,6 +28,7 @@ import org.apache.solr.SolrTestCase;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.junit.Test;

152
solr/solrj/src/test/org/apache/solr/client/solrj/routing/RequestReplicaListTransformerGeneratorTest.java Normal file
View File

@ -0,0 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.routing;
import java.util.ArrayList;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.junit.Test;
public class RequestReplicaListTransformerGeneratorTest extends SolrTestCaseJ4 {
@Test
public void testNodePreferenceRulesBase() {
RequestReplicaListTransformerGenerator generator = new RequestReplicaListTransformerGenerator();
ModifiableSolrParams params = new ModifiableSolrParams();
List<Replica> replicas = getBasicReplicaList();
String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
params.add("routingPreference", "0");
params.add(ShardParams.SHARDS_PREFERENCE, rulesParam);
ReplicaListTransformer rlt = generator.getReplicaListTransformer(params);
rlt.transform(replicas);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node2", replicas.get(1).getNodeName());
assertEquals("node3", replicas.get(2).getNodeName());
params.set("routingPreference", "1");
rlt = generator.getReplicaListTransformer(params);
rlt.transform(replicas);
assertEquals("node2", replicas.get(0).getNodeName());
assertEquals("node3", replicas.get(1).getNodeName());
assertEquals("node1", replicas.get(2).getNodeName());
params.set("routingPreference", "2");
rlt = generator.getReplicaListTransformer(params);
rlt.transform(replicas);
assertEquals("node3", replicas.get(0).getNodeName());
assertEquals("node1", replicas.get(1).getNodeName());
assertEquals("node2", replicas.get(2).getNodeName());
params.set("routingPreference", "3");
rlt = generator.getReplicaListTransformer(params);
rlt.transform(replicas);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node2", replicas.get(1).getNodeName());
assertEquals("node3", replicas.get(2).getNodeName());
}
@SuppressWarnings("unchecked")
@Test
public void replicaTypeAndReplicaBase() {
RequestReplicaListTransformerGenerator generator = new RequestReplicaListTransformerGenerator();
ModifiableSolrParams params = new ModifiableSolrParams();
List<Replica> replicas = getBasicReplicaList();
// Add a replica so that sorting by replicaType:TLOG can cause a tie
replicas.add(
new Replica(
"node4",
map(
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node4",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "TLOG"
)
)
);
// replicaType and replicaBase combined rule param
String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
params.add("routingPreference", "0");
params.add(ShardParams.SHARDS_PREFERENCE, rulesParam);
ReplicaListTransformer rlt = generator.getReplicaListTransformer(params);
rlt.transform(replicas);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node2", replicas.get(1).getNodeName());
assertEquals("node4", replicas.get(2).getNodeName());
assertEquals("node3", replicas.get(3).getNodeName());
params.set("routingPreference", "1");
rlt = generator.getReplicaListTransformer(params);
rlt.transform(replicas);
assertEquals("node1", replicas.get(0).getNodeName());
assertEquals("node4", replicas.get(1).getNodeName());
assertEquals("node2", replicas.get(2).getNodeName());
assertEquals("node3", replicas.get(3).getNodeName());
}
@SuppressWarnings("unchecked")
private static List<Replica> getBasicReplicaList() {
List<Replica> replicas = new ArrayList<Replica>();
replicas.add(
new Replica(
"node1",
map(
ZkStateReader.BASE_URL_PROP, "http://host1:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node1",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "NRT"
)
)
);
replicas.add(
new Replica(
"node2",
map(
ZkStateReader.BASE_URL_PROP, "http://host2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node2",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "TLOG"
)
)
);
replicas.add(
new Replica(
"node3",
map(
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
ZkStateReader.NODE_NAME_PROP, "node3",
ZkStateReader.CORE_NAME_PROP, "collection1",
ZkStateReader.REPLICA_TYPE, "PULL"
)
)
);
return replicas;
}
}

2
solr/core/src/test/org/apache/solr/handler/component/ShufflingReplicaListTransformerTest.java → solr/solrj/src/test/org/apache/solr/client/solrj/routing/ShufflingReplicaListTransformerTest.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
package org.apache.solr.client.solrj.routing;
import java.util.ArrayList;
import java.util.Collections;