mirror of https://github.com/apache/lucene.git
SOLR-13865: Migrate replica routing code to SolrJ (#974)
* [SOLR-13865] Migrate replica routing code to solrJ * Added a CommonTestInjection class. * Fixing imports. * Reverted extraneous streaming changes. * Fix precommit errors. * Changing name of the RLTManager. * Splitting up existing tests. * Updated documentation. * Added solr/CHANGES.txt entry
This commit is contained in:
Houston Putman
Tomas Fernandez Lobbe
parent
3af4e6adc6
commit
fa27e476f7
|
|
@ -113,6 +113,8 @@ Improvements
|
|||
|
||||
* SOLR-13831: Support defining arbitrary autoscaling simulation scenarios. (ab)
|
||||
|
||||
* SOLR-13865: Move replica routing code to SolrJ. (Houston Putman via Tomas Fernandez-Lobbe)
|
||||
|
||||
|
||||
Optimizations
|
||||
---------------------
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ import org.apache.solr.common.cloud.DefaultZkCredentialsProvider;
|
|||
import org.apache.solr.common.cloud.DocCollection;
|
||||
import org.apache.solr.common.cloud.DocCollectionWatcher;
|
||||
import org.apache.solr.common.cloud.LiveNodesListener;
|
||||
import org.apache.solr.common.cloud.NodesSysPropsCacher;
|
||||
import org.apache.solr.common.cloud.OnReconnect;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.Replica.Type;
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import com.codahale.metrics.MetricRegistry;
|
|||
import com.codahale.metrics.Timer;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.CommonTestInjection;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
|
|
@ -48,7 +49,6 @@ import org.apache.solr.request.SolrQueryRequest;
|
|||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.security.AuthorizationContext;
|
||||
import org.apache.solr.security.PermissionNameProvider;
|
||||
import org.apache.solr.util.TestInjection;
|
||||
import org.apache.solr.util.stats.MetricUtils;
|
||||
|
||||
/**
|
||||
|
|
@ -70,7 +70,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
|
|||
|
||||
private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
|
||||
private CoreContainer cc;
|
||||
private final Map<String, String> injectedSysProps = TestInjection.injectAdditionalProps();
|
||||
private final Map<String, String> injectedSysProps = CommonTestInjection.injectAdditionalProps();
|
||||
|
||||
public MetricsHandler() {
|
||||
this.metricManager = null;
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ import org.apache.solr.client.solrj.SolrServerException;
|
|||
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
|
||||
import org.apache.solr.client.solrj.impl.Http2SolrClient;
|
||||
import org.apache.solr.client.solrj.impl.LBSolrClient;
|
||||
import org.apache.solr.client.solrj.routing.ReplicaListTransformer;
|
||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||
import org.apache.solr.client.solrj.util.ClientUtils;
|
||||
import org.apache.solr.cloud.CloudDescriptor;
|
||||
|
|
|
|||
|
|
@ -16,20 +16,13 @@
|
|||
*/
|
||||
package org.apache.solr.handler.component;
|
||||
|
||||
import static org.apache.solr.util.stats.InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
|
|
@ -41,6 +34,8 @@ import java.util.concurrent.ExecutorService;
|
|||
import java.util.concurrent.SynchronousQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
|
|
@ -51,16 +46,16 @@ import org.apache.solr.client.solrj.impl.HttpClientUtil;
|
|||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
|
||||
import org.apache.solr.client.solrj.impl.LBSolrClient;
|
||||
import org.apache.solr.client.solrj.impl.PreferenceRule;
|
||||
import org.apache.solr.client.solrj.routing.AffinityReplicaListTransformerFactory;
|
||||
import org.apache.solr.client.solrj.routing.ReplicaListTransformer;
|
||||
import org.apache.solr.client.solrj.routing.ReplicaListTransformerFactory;
|
||||
import org.apache.solr.client.solrj.routing.RequestReplicaListTransformerGenerator;
|
||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||
import org.apache.solr.cloud.NodesSysPropsCacher;
|
||||
import org.apache.solr.cloud.ZkController;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.ExecutorUtil;
|
||||
|
|
@ -83,6 +78,8 @@ import org.apache.solr.util.stats.MetricUtils;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.solr.util.stats.InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES;
|
||||
|
||||
public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized, SolrMetricProducer {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
private static final String DEFAULT_SCHEME = "http";
|
||||
|
|
@ -124,7 +121,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
|
|||
|
||||
protected final Random r = new Random();
|
||||
|
||||
private final ReplicaListTransformer shufflingReplicaListTransformer = new ShufflingReplicaListTransformer(r);
|
||||
private RequestReplicaListTransformerGenerator requestReplicaListTransformerGenerator = new RequestReplicaListTransformerGenerator();
|
||||
|
||||
// URL scheme to be used in distributed search.
|
||||
static final String INIT_URL_SCHEME = "urlScheme";
|
||||
|
|
@ -224,6 +221,8 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
|
|||
|
||||
private void initReplicaListTransformers(NamedList routingConfig) {
|
||||
String defaultRouting = null;
|
||||
ReplicaListTransformerFactory stableRltFactory = null;
|
||||
ReplicaListTransformerFactory defaultRltFactory;
|
||||
if (routingConfig != null && routingConfig.size() > 0) {
|
||||
Iterator<Entry<String,?>> iter = routingConfig.iterator();
|
||||
do {
|
||||
|
|
@ -240,21 +239,22 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
|
|||
case ShardParams.REPLICA_STABLE:
|
||||
NamedList<?> c = getNamedList(e.getValue());
|
||||
defaultRouting = checkDefaultReplicaListTransformer(c, key, defaultRouting);
|
||||
this.stableRltFactory = new AffinityReplicaListTransformerFactory(c);
|
||||
stableRltFactory = new AffinityReplicaListTransformerFactory(c);
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("invalid replica routing spec name: " + key);
|
||||
}
|
||||
} while (iter.hasNext());
|
||||
}
|
||||
if (this.stableRltFactory == null) {
|
||||
this.stableRltFactory = new AffinityReplicaListTransformerFactory();
|
||||
if (stableRltFactory == null) {
|
||||
stableRltFactory = new AffinityReplicaListTransformerFactory();
|
||||
}
|
||||
if (ShardParams.REPLICA_STABLE.equals(defaultRouting)) {
|
||||
this.defaultRltFactory = this.stableRltFactory;
|
||||
defaultRltFactory = stableRltFactory;
|
||||
} else {
|
||||
this.defaultRltFactory = this.randomRltFactory;
|
||||
defaultRltFactory = RequestReplicaListTransformerGenerator.RANDOM_RLTF;
|
||||
}
|
||||
this.requestReplicaListTransformerGenerator = new RequestReplicaListTransformerGenerator(defaultRltFactory, stableRltFactory);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -269,7 +269,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
|
|||
String strategy = getParameter(args, "metricNameStrategy", UpdateShardHandlerConfig.DEFAULT_METRICNAMESTRATEGY, sb);
|
||||
this.metricNameStrategy = KNOWN_METRIC_NAME_STRATEGIES.get(strategy);
|
||||
if (this.metricNameStrategy == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR,
|
||||
"Unknown metricNameStrategy: " + strategy + " found. Must be one of: " + KNOWN_METRIC_NAME_STRATEGIES.keySet());
|
||||
}
|
||||
|
||||
|
|
@ -409,282 +409,25 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
|
|||
return urls;
|
||||
}
|
||||
|
||||
/**
|
||||
* A distributed request is made via {@link LBSolrClient} to the first live server in the URL list.
|
||||
* This means it is just as likely to choose current host as any of the other hosts.
|
||||
* This function makes sure that the cores are sorted according to the given list of preferences.
|
||||
* E.g. If all nodes prefer local cores then a bad/heavily-loaded node will receive less requests from
|
||||
* healthy nodes. This will help prevent a distributed deadlock or timeouts in all the healthy nodes due
|
||||
* to one bad node.
|
||||
*
|
||||
* Optional final preferenceRule is *not* used for pairwise sorting, but instead defines how "equivalent"
|
||||
* replicas will be ordered (the base ordering). Defaults to "random"; may specify "stable".
|
||||
*/
|
||||
static class NodePreferenceRulesComparator implements Comparator<Object> {
|
||||
|
||||
private final SolrQueryRequest request;
|
||||
private final NodesSysPropsCacher sysPropsCache;
|
||||
private final String nodeName;
|
||||
private final List<PreferenceRule> sortRules;
|
||||
private final List<PreferenceRule> preferenceRules;
|
||||
private String localHostAddress = null;
|
||||
private final ReplicaListTransformer baseReplicaListTransformer;
|
||||
|
||||
public NodePreferenceRulesComparator(final List<PreferenceRule> preferenceRules, final SolrQueryRequest request,
|
||||
final ReplicaListTransformerFactory defaultRltFactory, final ReplicaListTransformerFactory randomRltFactory,
|
||||
final ReplicaListTransformerFactory stableRltFactory) {
|
||||
this.request = request;
|
||||
final SolrCore core; // explicit check for null core (temporary?, for tests)
|
||||
if (request != null && (core = request.getCore()) != null && core.getCoreContainer().getZkController() != null) {
|
||||
ZkController zkController = request.getCore().getCoreContainer().getZkController();
|
||||
sysPropsCache = zkController.getSysPropsCacher();
|
||||
nodeName = zkController.getNodeName();
|
||||
} else {
|
||||
sysPropsCache = null;
|
||||
nodeName = null;
|
||||
}
|
||||
this.preferenceRules = preferenceRules;
|
||||
final int maxIdx = preferenceRules.size() - 1;
|
||||
final PreferenceRule lastRule = preferenceRules.get(maxIdx);
|
||||
if (!ShardParams.SHARDS_PREFERENCE_REPLICA_BASE.equals(lastRule.name)) {
|
||||
this.sortRules = preferenceRules;
|
||||
this.baseReplicaListTransformer = defaultRltFactory.getInstance(null, request, randomRltFactory);
|
||||
} else {
|
||||
if (maxIdx == 0) {
|
||||
this.sortRules = null;
|
||||
} else {
|
||||
this.sortRules = preferenceRules.subList(0, maxIdx);
|
||||
}
|
||||
String[] parts = lastRule.value.split(":", 2);
|
||||
switch (parts[0]) {
|
||||
case ShardParams.REPLICA_RANDOM:
|
||||
this.baseReplicaListTransformer = randomRltFactory.getInstance(parts.length == 1 ? null : parts[1], request, null);
|
||||
break;
|
||||
case ShardParams.REPLICA_STABLE:
|
||||
this.baseReplicaListTransformer = stableRltFactory.getInstance(parts.length == 1 ? null : parts[1], request, randomRltFactory);
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Invalid base replica order spec");
|
||||
}
|
||||
}
|
||||
}
|
||||
private static final ReplicaListTransformer NOOP_RLT = (List<?> choices) -> { /* noop */ };
|
||||
private static final ReplicaListTransformerFactory NOOP_RLTF = (String configSpec, SolrQueryRequest request,
|
||||
ReplicaListTransformerFactory fallback) -> NOOP_RLT;
|
||||
/**
|
||||
* For compatibility with tests, which expect this constructor to have no effect on the *base* order.
|
||||
*/
|
||||
NodePreferenceRulesComparator(final List<PreferenceRule> sortRules, final SolrQueryRequest request) {
|
||||
this(sortRules, request, NOOP_RLTF, null, null);
|
||||
}
|
||||
|
||||
public ReplicaListTransformer getBaseReplicaListTransformer() {
|
||||
return baseReplicaListTransformer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compare(Object left, Object right) {
|
||||
if (this.sortRules != null) {
|
||||
for (PreferenceRule preferenceRule: this.sortRules) {
|
||||
final boolean lhs;
|
||||
final boolean rhs;
|
||||
switch (preferenceRule.name) {
|
||||
case ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE:
|
||||
lhs = hasReplicaType(left, preferenceRule.value);
|
||||
rhs = hasReplicaType(right, preferenceRule.value);
|
||||
break;
|
||||
case ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION:
|
||||
lhs = hasCoreUrlPrefix(left, preferenceRule.value);
|
||||
rhs = hasCoreUrlPrefix(right, preferenceRule.value);
|
||||
break;
|
||||
case ShardParams.SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP:
|
||||
if (sysPropsCache == null) {
|
||||
throw new IllegalArgumentException("Unable to get the NodesSysPropsCacher" +
|
||||
" on sorting replicas by preference:"+ preferenceRule.value);
|
||||
}
|
||||
lhs = hasSameMetric(left, preferenceRule.value);
|
||||
rhs = hasSameMetric(right, preferenceRule.value);
|
||||
break;
|
||||
case ShardParams.SHARDS_PREFERENCE_REPLICA_BASE:
|
||||
throw new IllegalArgumentException("only one base replica order may be specified in "
|
||||
+ ShardParams.SHARDS_PREFERENCE + ", and it must be specified last");
|
||||
default:
|
||||
throw new IllegalArgumentException("Invalid " + ShardParams.SHARDS_PREFERENCE + " type: " + preferenceRule.name);
|
||||
}
|
||||
if (lhs != rhs) {
|
||||
return lhs ? -1 : +1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private boolean hasSameMetric(Object o, String metricTag) {
|
||||
if (!(o instanceof Replica)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Collection<String> tags = Collections.singletonList(metricTag);
|
||||
String otherNodeName = ((Replica) o).getNodeName();
|
||||
Map<String, Object> currentNodeMetric = sysPropsCache.getSysProps(nodeName, tags);
|
||||
Map<String, Object> otherNodeMetric = sysPropsCache.getSysProps(otherNodeName, tags);
|
||||
return currentNodeMetric.equals(otherNodeMetric);
|
||||
}
|
||||
|
||||
private boolean hasCoreUrlPrefix(Object o, String prefix) {
|
||||
final String s;
|
||||
if (o instanceof String) {
|
||||
s = (String)o;
|
||||
}
|
||||
else if (o instanceof Replica) {
|
||||
s = ((Replica)o).getCoreUrl();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
if (prefix.equals(ShardParams.REPLICA_LOCAL)) {
|
||||
if (null == localHostAddress) {
|
||||
final ZkController zkController = this.request.getCore().getCoreContainer().getZkController();
|
||||
localHostAddress = zkController != null ? zkController.getBaseUrl() : "";
|
||||
if (localHostAddress.isEmpty()) {
|
||||
log.warn("Couldn't determine current host address for sorting of local replicas");
|
||||
}
|
||||
}
|
||||
if (!localHostAddress.isEmpty()) {
|
||||
if (s.startsWith(localHostAddress)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (s.startsWith(prefix)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
private static boolean hasReplicaType(Object o, String preferred) {
|
||||
if (!(o instanceof Replica)) {
|
||||
return false;
|
||||
}
|
||||
final String s = ((Replica)o).getType().toString();
|
||||
return s.equals(preferred);
|
||||
}
|
||||
}
|
||||
|
||||
private final ReplicaListTransformerFactory randomRltFactory = (String configSpec, SolrQueryRequest request,
|
||||
ReplicaListTransformerFactory fallback) -> shufflingReplicaListTransformer;
|
||||
private ReplicaListTransformerFactory stableRltFactory;
|
||||
private ReplicaListTransformerFactory defaultRltFactory;
|
||||
|
||||
/**
|
||||
* Private class responsible for applying pairwise sort based on inherent replica attributes,
|
||||
* and subsequently reordering any equivalent replica sets according to behavior specified
|
||||
* by the baseReplicaListTransformer.
|
||||
*/
|
||||
private static final class TopLevelReplicaListTransformer implements ReplicaListTransformer {
|
||||
|
||||
private final NodePreferenceRulesComparator replicaComp;
|
||||
private final ReplicaListTransformer baseReplicaListTransformer;
|
||||
|
||||
public TopLevelReplicaListTransformer(NodePreferenceRulesComparator replicaComp, ReplicaListTransformer baseReplicaListTransformer) {
|
||||
this.replicaComp = replicaComp;
|
||||
this.baseReplicaListTransformer = baseReplicaListTransformer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void transform(List<?> choices) {
|
||||
if (choices.size() > 1) {
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Applying the following sorting preferences to replicas: {}",
|
||||
Arrays.toString(replicaComp.preferenceRules.toArray()));
|
||||
}
|
||||
|
||||
// First, sort according to comparator rules.
|
||||
try {
|
||||
choices.sort(replicaComp);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
throw new SolrException(
|
||||
SolrException.ErrorCode.BAD_REQUEST,
|
||||
iae.getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
// Next determine all boundaries between replicas ranked as "equivalent" by the comparator
|
||||
Iterator<?> iter = choices.iterator();
|
||||
Object prev = iter.next();
|
||||
Object current;
|
||||
int idx = 1;
|
||||
int boundaryCount = 0;
|
||||
int[] boundaries = new int[choices.size() - 1];
|
||||
do {
|
||||
current = iter.next();
|
||||
if (replicaComp.compare(prev, current) != 0) {
|
||||
boundaries[boundaryCount++] = idx;
|
||||
}
|
||||
prev = current;
|
||||
idx++;
|
||||
} while (iter.hasNext());
|
||||
|
||||
// Finally inspect boundaries to apply base transformation, where necessary (separate phase to avoid ConcurrentModificationException)
|
||||
int startIdx = 0;
|
||||
int endIdx;
|
||||
for (int i = 0; i < boundaryCount; i++) {
|
||||
endIdx = boundaries[i];
|
||||
if (endIdx - startIdx > 1) {
|
||||
baseReplicaListTransformer.transform(choices.subList(startIdx, endIdx));
|
||||
}
|
||||
startIdx = endIdx;
|
||||
}
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Applied sorting preferences to replica list: {}",
|
||||
Arrays.toString(choices.toArray()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected ReplicaListTransformer getReplicaListTransformer(final SolrQueryRequest req) {
|
||||
final SolrParams params = req.getParams();
|
||||
final SolrCore core = req.getCore(); // explicit check for null core (temporary?, for tests)
|
||||
ZkController zkController = core == null ? null : core.getCoreContainer().getZkController();
|
||||
String defaultShardPreference = "";
|
||||
if (zkController != null) {
|
||||
defaultShardPreference = zkController.getZkStateReader().getClusterProperties()
|
||||
return requestReplicaListTransformerGenerator.getReplicaListTransformer(
|
||||
params,
|
||||
zkController.getZkStateReader().getClusterProperties()
|
||||
.getOrDefault(ZkStateReader.DEFAULT_SHARD_PREFERENCES, "")
|
||||
.toString();
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
final boolean preferLocalShards = params.getBool(CommonParams.PREFER_LOCAL_SHARDS, false);
|
||||
final String shardsPreferenceSpec = params.get(ShardParams.SHARDS_PREFERENCE, defaultShardPreference);
|
||||
|
||||
if (preferLocalShards || !shardsPreferenceSpec.isEmpty()) {
|
||||
if (preferLocalShards && !shardsPreferenceSpec.isEmpty()) {
|
||||
throw new SolrException(
|
||||
SolrException.ErrorCode.BAD_REQUEST,
|
||||
"preferLocalShards is deprecated and must not be used with shards.preference"
|
||||
.toString(),
|
||||
zkController.getNodeName(),
|
||||
zkController.getBaseUrl(),
|
||||
zkController.getSysPropsCacher()
|
||||
);
|
||||
}
|
||||
List<PreferenceRule> preferenceRules = PreferenceRule.from(shardsPreferenceSpec);
|
||||
if (preferLocalShards) {
|
||||
preferenceRules.add(new PreferenceRule(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION, ShardParams.REPLICA_LOCAL));
|
||||
}
|
||||
|
||||
NodePreferenceRulesComparator replicaComp = new NodePreferenceRulesComparator(preferenceRules, req,
|
||||
defaultRltFactory, randomRltFactory, stableRltFactory);
|
||||
ReplicaListTransformer baseReplicaListTransformer = replicaComp.getBaseReplicaListTransformer();
|
||||
if (replicaComp.sortRules == null) {
|
||||
// only applying base transformation
|
||||
return baseReplicaListTransformer;
|
||||
} else {
|
||||
return new TopLevelReplicaListTransformer(replicaComp, baseReplicaListTransformer);
|
||||
return requestReplicaListTransformerGenerator.getReplicaListTransformer(params);
|
||||
}
|
||||
}
|
||||
|
||||
return defaultRltFactory.getInstance(null, req, randomRltFactory);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new completion service for use by a single set of distributed requests.
|
||||
*/
|
||||
|
|
@ -753,10 +496,10 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
|
|||
url = new URL(hostUrl);
|
||||
}
|
||||
} catch (MalformedURLException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist, e);
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist, e);
|
||||
}
|
||||
if (url.getHost() == null || url.getPort() < 0) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist);
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist);
|
||||
}
|
||||
return url.getHost() + ":" + url.getPort();
|
||||
}).collect(Collectors.toSet());
|
||||
|
|
@ -804,10 +547,10 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
|
|||
url = new URL(shardUrl);
|
||||
}
|
||||
} catch (MalformedURLException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue, e);
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue, e);
|
||||
}
|
||||
if (url.getHost() == null || url.getPort() < 0) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue);
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue);
|
||||
}
|
||||
if (!localWhitelistHosts.contains(url.getHost() + ":" + url.getPort())) {
|
||||
log.warn("The '"+ShardParams.SHARDS+"' parameter value '"+shardsParamValue+"' contained value(s) not on the shards whitelist ("+localWhitelistHosts+"), shardUrl:" + shardUrl);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ import java.lang.invoke.MethodHandles;
|
|||
import java.lang.reflect.Method;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
import java.util.Timer;
|
||||
|
|
@ -147,8 +146,6 @@ public class TestInjection {
|
|||
|
||||
public volatile static boolean uifOutOfMemoryError = false;
|
||||
|
||||
public volatile static Map<String, String> additionalSystemProps = null;
|
||||
|
||||
private volatile static CountDownLatch notifyPauseForeverDone = new CountDownLatch(1);
|
||||
|
||||
public static void notifyPauseForeverDone() {
|
||||
|
|
@ -157,7 +154,6 @@ public class TestInjection {
|
|||
}
|
||||
|
||||
public static void reset() {
|
||||
additionalSystemProps = null;
|
||||
nonGracefullClose = null;
|
||||
failReplicaRequests = null;
|
||||
failUpdateRequests = null;
|
||||
|
|
@ -494,10 +490,6 @@ public class TestInjection {
|
|||
return true;
|
||||
}
|
||||
|
||||
public static Map<String,String> injectAdditionalProps() {
|
||||
return additionalSystemProps;
|
||||
}
|
||||
|
||||
public static boolean injectUIFOutOfMemoryError() {
|
||||
if (uifOutOfMemoryError ) {
|
||||
throw new OutOfMemoryError("Test Injection");
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ import org.apache.solr.common.cloud.Slice;
|
|||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.util.CommonTestInjection;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.common.util.TimeSource;
|
||||
|
|
@ -67,14 +68,14 @@ public class RoutingToNodesWithPropertiesTest extends SolrCloudTestCase {
|
|||
|
||||
@Before
|
||||
public void setupCluster() throws Exception {
|
||||
TestInjection.additionalSystemProps = ImmutableMap.of("zone", "us-west1");
|
||||
CommonTestInjection.setAdditionalProps(ImmutableMap.of("zone", "us-west1"));
|
||||
configureCluster(2)
|
||||
.withSolrXml(TEST_PATH().resolve("solr-trackingshardhandler.xml"))
|
||||
.addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
|
||||
.configure();
|
||||
|
||||
zone1Nodes.addAll(cluster.getJettySolrRunners().stream().map(JettySolrRunner::getNodeName).collect(Collectors.toSet()));
|
||||
TestInjection.additionalSystemProps = ImmutableMap.of("zone", "us-west2");
|
||||
CommonTestInjection.setAdditionalProps(ImmutableMap.of("zone", "us-west2"));
|
||||
zone2Nodes.add(cluster.startJettySolrRunner().getNodeName());
|
||||
zone2Nodes.add(cluster.startJettySolrRunner().getNodeName());
|
||||
|
||||
|
|
@ -114,9 +115,9 @@ public class RoutingToNodesWithPropertiesTest extends SolrCloudTestCase {
|
|||
assertEquals("us-west1", map.get(PROP_NAME));
|
||||
}
|
||||
|
||||
for (String zone1Node: zone2Nodes) {
|
||||
for (String zone2Node: zone2Nodes) {
|
||||
NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
|
||||
Map<String, Object> map = nodeStateProvider.getNodeValues(zone1Node, Collections.singletonList(PROP_NAME));
|
||||
Map<String, Object> map = nodeStateProvider.getNodeValues(zone2Node, Collections.singletonList(PROP_NAME));
|
||||
assertEquals("us-west2", map.get(PROP_NAME));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ import java.nio.file.Path;
|
|||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
|
@ -28,19 +27,11 @@ import java.util.Set;
|
|||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.client.solrj.impl.LBSolrClient;
|
||||
import org.apache.solr.client.solrj.impl.PreferenceRule;
|
||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.core.PluginInfo;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.handler.component.HttpShardHandlerFactory.WhitelistHostChecker;
|
||||
import org.apache.solr.request.SolrQueryRequestBase;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
|
@ -121,208 +112,6 @@ public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 {
|
|||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testNodePreferenceRulesBase() throws Exception {
|
||||
SolrCore testCore = null;
|
||||
HttpShardHandlerFactory fac = new HttpShardHandlerFactory();
|
||||
fac.init(new PluginInfo(null, Collections.EMPTY_MAP));
|
||||
SolrQueryRequestBase req;
|
||||
NamedList<String> params = new NamedList<>();
|
||||
List<Replica> replicas = getBasicReplicaList();
|
||||
|
||||
String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
|
||||
|
||||
params.add("routingPreference", "0");
|
||||
params.add(ShardParams.SHARDS_PREFERENCE, rulesParam);
|
||||
|
||||
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
|
||||
ReplicaListTransformer rlt = fac.getReplicaListTransformer(req);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node2", replicas.get(1).getNodeName());
|
||||
assertEquals("node3", replicas.get(2).getNodeName());
|
||||
req.close();
|
||||
|
||||
params.setVal(0, "1");
|
||||
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
|
||||
rlt = fac.getReplicaListTransformer(req);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node2", replicas.get(0).getNodeName());
|
||||
assertEquals("node3", replicas.get(1).getNodeName());
|
||||
assertEquals("node1", replicas.get(2).getNodeName());
|
||||
req.close();
|
||||
|
||||
params.setVal(0, "2");
|
||||
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
|
||||
rlt = fac.getReplicaListTransformer(req);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node3", replicas.get(0).getNodeName());
|
||||
assertEquals("node1", replicas.get(1).getNodeName());
|
||||
assertEquals("node2", replicas.get(2).getNodeName());
|
||||
req.close();
|
||||
|
||||
params.setVal(0, "3");
|
||||
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
|
||||
rlt = fac.getReplicaListTransformer(req);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node2", replicas.get(1).getNodeName());
|
||||
assertEquals("node3", replicas.get(2).getNodeName());
|
||||
req.close();
|
||||
|
||||
// Add a replica so that sorting by replicaType:TLOG can cause a tie
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node4",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node4",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "TLOG"
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// replicaType and replicaBase combined rule param
|
||||
rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
|
||||
|
||||
params.setVal(0, "0");
|
||||
params.setVal(1, rulesParam);
|
||||
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
|
||||
rlt = fac.getReplicaListTransformer(req);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node2", replicas.get(1).getNodeName());
|
||||
assertEquals("node4", replicas.get(2).getNodeName());
|
||||
assertEquals("node3", replicas.get(3).getNodeName());
|
||||
req.close();
|
||||
|
||||
params.setVal(0, "1");
|
||||
req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
|
||||
rlt = fac.getReplicaListTransformer(req);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node4", replicas.get(1).getNodeName());
|
||||
assertEquals("node2", replicas.get(2).getNodeName());
|
||||
assertEquals("node3", replicas.get(3).getNodeName());
|
||||
req.close();
|
||||
fac.close();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<Replica> getBasicReplicaList() {
|
||||
List<Replica> replicas = new ArrayList<Replica>();
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node1",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host1:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node1",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "NRT"
|
||||
)
|
||||
)
|
||||
);
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node2",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node2",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "TLOG"
|
||||
)
|
||||
)
|
||||
);
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node3",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node3",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "PULL"
|
||||
)
|
||||
)
|
||||
);
|
||||
return replicas;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testNodePreferenceRulesComparator() throws Exception {
|
||||
List<Replica> replicas = getBasicReplicaList();
|
||||
|
||||
// Simple replica type rule
|
||||
List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG");
|
||||
HttpShardHandlerFactory.NodePreferenceRulesComparator comparator =
|
||||
new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
|
||||
replicas.sort(comparator);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node2", replicas.get(1).getNodeName());
|
||||
|
||||
// Another simple replica type rule
|
||||
rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT");
|
||||
comparator = new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
|
||||
replicas.sort(comparator);
|
||||
assertEquals("node2", replicas.get(0).getNodeName());
|
||||
assertEquals("node1", replicas.get(1).getNodeName());
|
||||
|
||||
// replicaLocation rule
|
||||
rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2:8983");
|
||||
comparator = new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
|
||||
replicas.sort(comparator);
|
||||
assertEquals("node2", replicas.get(0).getNodeName());
|
||||
assertEquals("node1", replicas.get(1).getNodeName());
|
||||
|
||||
// Add a replica so that sorting by replicaType:TLOG can cause a tie
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node4",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node4",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "TLOG"
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// replicaType and replicaLocation combined rule
|
||||
rules = PreferenceRule.from(
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2_2");
|
||||
comparator = new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
|
||||
replicas.sort(comparator);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node4", replicas.get(1).getNodeName());
|
||||
assertEquals("node2", replicas.get(2).getNodeName());
|
||||
assertEquals("node3", replicas.get(3).getNodeName());
|
||||
|
||||
// Bad rule
|
||||
|
||||
try {
|
||||
rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE);
|
||||
fail();
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertEquals("Invalid shards.preference rule: " + ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE, e.getMessage());
|
||||
}
|
||||
|
||||
// Unknown rule
|
||||
rules = PreferenceRule.from("badRule:test");
|
||||
try {
|
||||
comparator = new HttpShardHandlerFactory.NodePreferenceRulesComparator(rules, null);
|
||||
replicas.sort(comparator);
|
||||
fail();
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertEquals("Invalid shards.preference type: badRule", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getShardsWhitelist() throws Exception {
|
||||
System.setProperty(SHARDS_WHITELIST, "http://abc:8983/,http://def:8984/,");
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.component;
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
|
|
@ -24,14 +24,13 @@ import java.util.ListIterator;
|
|||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.Hash;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
|
||||
/**
|
||||
* Allows better caching by establishing deterministic evenly-distributed replica routing preferences according to
|
||||
* either explicitly configured hash routing parameter, or the hash of a query parameter (configurable, usually related
|
||||
* to the main query).
|
||||
*/
|
||||
class AffinityReplicaListTransformer implements ReplicaListTransformer {
|
||||
public class AffinityReplicaListTransformer implements ReplicaListTransformer {
|
||||
|
||||
private final int routingDividend;
|
||||
|
||||
|
|
@ -47,17 +46,16 @@ class AffinityReplicaListTransformer implements ReplicaListTransformer {
|
|||
*
|
||||
* @param dividendParam int param to be used directly for mod-based routing
|
||||
* @param hashParam String param to be hashed into an int for mod-based routing
|
||||
* @param req the request from which param values will be drawn
|
||||
* @param requestParams the parameters of the Solr request
|
||||
* @return null if specified routing vals are not able to be parsed properly
|
||||
*/
|
||||
public static ReplicaListTransformer getInstance(String dividendParam, String hashParam, SolrQueryRequest req) {
|
||||
SolrParams params = req.getOriginalParams();
|
||||
public static ReplicaListTransformer getInstance(String dividendParam, String hashParam, SolrParams requestParams) {
|
||||
Integer dividendVal;
|
||||
if (dividendParam != null && (dividendVal = params.getInt(dividendParam)) != null) {
|
||||
if (dividendParam != null && (dividendVal = requestParams.getInt(dividendParam)) != null) {
|
||||
return new AffinityReplicaListTransformer(dividendVal);
|
||||
}
|
||||
String hashVal;
|
||||
if (hashParam != null && (hashVal = params.get(hashParam)) != null && !hashVal.isEmpty()) {
|
||||
if (hashParam != null && (hashVal = requestParams.get(hashParam)) != null && !hashVal.isEmpty()) {
|
||||
return new AffinityReplicaListTransformer(hashVal);
|
||||
} else {
|
||||
return null;
|
||||
|
|
@ -14,12 +14,12 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.component;
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
|
||||
/**
|
||||
* Factory for constructing an {@link AffinityReplicaListTransformer} that reorders replica routing
|
||||
|
|
@ -27,12 +27,12 @@ import org.apache.solr.request.SolrQueryRequest;
|
|||
*
|
||||
* Default names of params that contain the values by which routing is determined may be configured
|
||||
* at the time of {@link AffinityReplicaListTransformerFactory} construction, and may be
|
||||
* overridden by the config spec passed to {@link #getInstance(String, SolrQueryRequest, ReplicaListTransformerFactory)}
|
||||
* overridden by the config spec passed to {@link #getInstance(String, SolrParams, ReplicaListTransformerFactory)}
|
||||
*
|
||||
* If no defaultHashParam name is specified at time of factory construction, the routing dividend will
|
||||
* be derived by hashing the {@link String} value of the {@link CommonParams#Q} param.
|
||||
*/
|
||||
class AffinityReplicaListTransformerFactory implements ReplicaListTransformerFactory {
|
||||
public class AffinityReplicaListTransformerFactory implements ReplicaListTransformerFactory {
|
||||
private final String defaultDividendParam;
|
||||
private final String defaultHashParam;
|
||||
|
||||
|
|
@ -68,24 +68,24 @@ class AffinityReplicaListTransformerFactory implements ReplicaListTransformerFac
|
|||
}
|
||||
|
||||
@Override
|
||||
public ReplicaListTransformer getInstance(String configSpec, SolrQueryRequest request, ReplicaListTransformerFactory fallback) {
|
||||
public ReplicaListTransformer getInstance(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback) {
|
||||
ReplicaListTransformer rlt;
|
||||
if (configSpec == null) {
|
||||
rlt = AffinityReplicaListTransformer.getInstance(defaultDividendParam, defaultHashParam, request);
|
||||
rlt = AffinityReplicaListTransformer.getInstance(defaultDividendParam, defaultHashParam, requestParams);
|
||||
} else {
|
||||
String[] parts = configSpec.split(":", 2);
|
||||
switch (parts[0]) {
|
||||
case ShardParams.ROUTING_DIVIDEND:
|
||||
rlt = AffinityReplicaListTransformer.getInstance(parts.length == 1 ? defaultDividendParam : parts[1], defaultHashParam, request);
|
||||
rlt = AffinityReplicaListTransformer.getInstance(parts.length == 1 ? defaultDividendParam : parts[1], defaultHashParam, requestParams);
|
||||
break;
|
||||
case ShardParams.ROUTING_HASH:
|
||||
rlt = AffinityReplicaListTransformer.getInstance(null, parts.length == 1 ? defaultHashParam : parts[1], request);
|
||||
rlt = AffinityReplicaListTransformer.getInstance(null, parts.length == 1 ? defaultHashParam : parts[1], requestParams);
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Invalid routing spec: \"" + configSpec + '"');
|
||||
}
|
||||
}
|
||||
return rlt != null ? rlt : fallback.getInstance(null, request, null);
|
||||
return rlt != null ? rlt : fallback.getInstance(null, requestParams, null);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.solr.common.StringUtils;
|
||||
import org.apache.solr.common.cloud.NodesSysPropsCacher;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
|
||||
/**
|
||||
* This comparator makes sure that the given replicas are sorted according to the given list of preferences.
|
||||
* E.g. If all nodes prefer local cores then a bad/heavily-loaded node will receive less requests from
|
||||
* healthy nodes. This will help prevent a distributed deadlock or timeouts in all the healthy nodes due
|
||||
* to one bad node.
|
||||
*
|
||||
* Optional final preferenceRule is *not* used for pairwise sorting, but instead defines how "equivalent"
|
||||
* replicas will be ordered (the base ordering). Defaults to "random"; may specify "stable".
|
||||
*/
|
||||
public class NodePreferenceRulesComparator implements Comparator<Object> {
|
||||
|
||||
private final NodesSysPropsCacher sysPropsCache;
|
||||
private final String nodeName;
|
||||
private final List<PreferenceRule> sortRules;
|
||||
private final List<PreferenceRule> preferenceRules;
|
||||
private final String localHostAddress;
|
||||
private final ReplicaListTransformer baseReplicaListTransformer;
|
||||
|
||||
public NodePreferenceRulesComparator(final List<PreferenceRule> preferenceRules, final SolrParams requestParams,
|
||||
final ReplicaListTransformerFactory defaultRltFactory, final ReplicaListTransformerFactory stableRltFactory) {
|
||||
this(preferenceRules, requestParams, null, null, null, defaultRltFactory, stableRltFactory);
|
||||
}
|
||||
|
||||
public NodePreferenceRulesComparator(final List<PreferenceRule> preferenceRules, final SolrParams requestParams,
|
||||
final String nodeName, final String localHostAddress, final NodesSysPropsCacher sysPropsCache,
|
||||
final ReplicaListTransformerFactory defaultRltFactory, final ReplicaListTransformerFactory stableRltFactory) {
|
||||
this.sysPropsCache = sysPropsCache;
|
||||
this.preferenceRules = preferenceRules;
|
||||
this.nodeName = nodeName;
|
||||
this.localHostAddress = localHostAddress;
|
||||
final int maxIdx = preferenceRules.size() - 1;
|
||||
final PreferenceRule lastRule = preferenceRules.get(maxIdx);
|
||||
if (!ShardParams.SHARDS_PREFERENCE_REPLICA_BASE.equals(lastRule.name)) {
|
||||
this.sortRules = preferenceRules;
|
||||
this.baseReplicaListTransformer = defaultRltFactory.getInstance(null, requestParams, RequestReplicaListTransformerGenerator.RANDOM_RLTF);
|
||||
} else {
|
||||
if (maxIdx == 0) {
|
||||
this.sortRules = null;
|
||||
} else {
|
||||
this.sortRules = preferenceRules.subList(0, maxIdx);
|
||||
}
|
||||
String[] parts = lastRule.value.split(":", 2);
|
||||
switch (parts[0]) {
|
||||
case ShardParams.REPLICA_RANDOM:
|
||||
this.baseReplicaListTransformer = RequestReplicaListTransformerGenerator.RANDOM_RLTF.getInstance(parts.length == 1 ? null : parts[1], requestParams, null);
|
||||
break;
|
||||
case ShardParams.REPLICA_STABLE:
|
||||
this.baseReplicaListTransformer = stableRltFactory.getInstance(parts.length == 1 ? null : parts[1], requestParams, RequestReplicaListTransformerGenerator.RANDOM_RLTF);
|
||||
break;
|
||||
default:
|
||||
throw new IllegalArgumentException("Invalid base replica order spec");
|
||||
}
|
||||
}
|
||||
}
|
||||
private static final ReplicaListTransformer NOOP_RLT = (List<?> choices) -> { /* noop */ };
|
||||
private static final ReplicaListTransformerFactory NOOP_RLTF =
|
||||
(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback) -> NOOP_RLT;
|
||||
/**
|
||||
* For compatibility with tests, which expect this constructor to have no effect on the *base* order.
|
||||
*/
|
||||
NodePreferenceRulesComparator(final List<PreferenceRule> sortRules, final SolrParams requestParams) {
|
||||
this(sortRules, requestParams, NOOP_RLTF, null);
|
||||
}
|
||||
|
||||
public ReplicaListTransformer getBaseReplicaListTransformer() {
|
||||
return baseReplicaListTransformer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compare(Object left, Object right) {
|
||||
if (this.sortRules != null) {
|
||||
for (PreferenceRule preferenceRule: this.sortRules) {
|
||||
final boolean lhs;
|
||||
final boolean rhs;
|
||||
switch (preferenceRule.name) {
|
||||
case ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE:
|
||||
lhs = hasReplicaType(left, preferenceRule.value);
|
||||
rhs = hasReplicaType(right, preferenceRule.value);
|
||||
break;
|
||||
case ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION:
|
||||
lhs = hasCoreUrlPrefix(left, preferenceRule.value);
|
||||
rhs = hasCoreUrlPrefix(right, preferenceRule.value);
|
||||
break;
|
||||
case ShardParams.SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP:
|
||||
if (sysPropsCache == null) {
|
||||
throw new IllegalArgumentException("Unable to get the NodesSysPropsCacher on sorting replicas by preference:"+ preferenceRule.value);
|
||||
}
|
||||
lhs = hasSameMetric(left, preferenceRule.value);
|
||||
rhs = hasSameMetric(right, preferenceRule.value);
|
||||
break;
|
||||
case ShardParams.SHARDS_PREFERENCE_REPLICA_BASE:
|
||||
throw new IllegalArgumentException("only one base replica order may be specified in "
|
||||
+ ShardParams.SHARDS_PREFERENCE + ", and it must be specified last");
|
||||
default:
|
||||
throw new IllegalArgumentException("Invalid " + ShardParams.SHARDS_PREFERENCE + " type: " + preferenceRule.name);
|
||||
}
|
||||
if (lhs != rhs) {
|
||||
return lhs ? -1 : +1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private boolean hasSameMetric(Object o, String metricTag) {
|
||||
if (!(o instanceof Replica)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Collection<String> tags = Collections.singletonList(metricTag);
|
||||
String otherNodeName = ((Replica) o).getNodeName();
|
||||
Map<String, Object> currentNodeMetric = sysPropsCache.getSysProps(nodeName, tags);
|
||||
Map<String, Object> otherNodeMetric = sysPropsCache.getSysProps(otherNodeName, tags);
|
||||
return currentNodeMetric.equals(otherNodeMetric);
|
||||
}
|
||||
|
||||
private boolean hasCoreUrlPrefix(Object o, String prefix) {
|
||||
final String s;
|
||||
if (o instanceof String) {
|
||||
s = (String)o;
|
||||
}
|
||||
else if (o instanceof Replica) {
|
||||
s = ((Replica)o).getCoreUrl();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
if (prefix.equals(ShardParams.REPLICA_LOCAL)) {
|
||||
return !StringUtils.isEmpty(localHostAddress) && s.startsWith(localHostAddress);
|
||||
} else {
|
||||
return s.startsWith(prefix);
|
||||
}
|
||||
}
|
||||
private static boolean hasReplicaType(Object o, String preferred) {
|
||||
if (!(o instanceof Replica)) {
|
||||
return false;
|
||||
}
|
||||
final String s = ((Replica)o).getType().toString();
|
||||
return s.equals(preferred);
|
||||
}
|
||||
|
||||
public List<PreferenceRule> getSortRules() {
|
||||
return sortRules;
|
||||
}
|
||||
|
||||
public List<PreferenceRule> getPreferenceRules() {
|
||||
return preferenceRules;
|
||||
}
|
||||
}
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.client.solrj.impl;
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.component;
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -14,21 +14,21 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.component;
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
|
||||
public interface ReplicaListTransformerFactory {
|
||||
|
||||
/**
|
||||
*
|
||||
* @param configSpec spec for dynamic configuration of ReplicaListTransformer
|
||||
* @param request the request for which the ReplicaListTransformer is being generated
|
||||
* @param requestParams the request parameters for which the ReplicaListTransformer is being generated
|
||||
* @param fallback used to generate fallback value; the getInstance() method of the specified fallback must not
|
||||
* return null; The fallback value itself may be null if this implementation is known to never return null (i.e., if
|
||||
* fallback will never be needed)
|
||||
* @return ReplicaListTransformer to be used for routing this request
|
||||
*/
|
||||
ReplicaListTransformer getInstance(String configSpec, SolrQueryRequest request, ReplicaListTransformerFactory fallback);
|
||||
ReplicaListTransformer getInstance(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.Arrays;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.cloud.NodesSysPropsCacher;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class RequestReplicaListTransformerGenerator {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
private static final Random r = new Random();
|
||||
|
||||
private static final ReplicaListTransformer shufflingReplicaListTransformer = new ShufflingReplicaListTransformer(r);
|
||||
public static final ReplicaListTransformerFactory RANDOM_RLTF =
|
||||
(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback) -> shufflingReplicaListTransformer;
|
||||
private final ReplicaListTransformerFactory stableRltFactory;
|
||||
private final ReplicaListTransformerFactory defaultRltFactory;
|
||||
|
||||
public RequestReplicaListTransformerGenerator() {
|
||||
this(RANDOM_RLTF);
|
||||
}
|
||||
|
||||
public RequestReplicaListTransformerGenerator(ReplicaListTransformerFactory defaultRltFactory) {
|
||||
this(defaultRltFactory, null);
|
||||
}
|
||||
|
||||
public RequestReplicaListTransformerGenerator(ReplicaListTransformerFactory defaultRltFactory, ReplicaListTransformerFactory stableRltFactory) {
|
||||
this.defaultRltFactory = defaultRltFactory;
|
||||
if (stableRltFactory == null) {
|
||||
this.stableRltFactory = new AffinityReplicaListTransformerFactory();
|
||||
} else {
|
||||
this.stableRltFactory = stableRltFactory;
|
||||
}
|
||||
}
|
||||
|
||||
public ReplicaListTransformer getReplicaListTransformer(final SolrParams requestParams) {
|
||||
return getReplicaListTransformer(requestParams, "");
|
||||
}
|
||||
|
||||
public ReplicaListTransformer getReplicaListTransformer(final SolrParams requestParams, String defaultShardPreferences) {
|
||||
return getReplicaListTransformer(requestParams, defaultShardPreferences, null, null, null);
|
||||
}
|
||||
|
||||
public ReplicaListTransformer getReplicaListTransformer(final SolrParams requestParams, String defaultShardPreferences, String nodeName, String localHostAddress, NodesSysPropsCacher sysPropsCacher) {
|
||||
@SuppressWarnings("deprecation")
|
||||
final boolean preferLocalShards = requestParams.getBool(CommonParams.PREFER_LOCAL_SHARDS, false);
|
||||
final String shardsPreferenceSpec = requestParams.get(ShardParams.SHARDS_PREFERENCE, defaultShardPreferences);
|
||||
|
||||
if (preferLocalShards || !shardsPreferenceSpec.isEmpty()) {
|
||||
if (preferLocalShards && !shardsPreferenceSpec.isEmpty()) {
|
||||
throw new SolrException(
|
||||
ErrorCode.BAD_REQUEST,
|
||||
"preferLocalShards is deprecated and must not be used with shards.preference"
|
||||
);
|
||||
}
|
||||
List<PreferenceRule> preferenceRules = PreferenceRule.from(shardsPreferenceSpec);
|
||||
if (preferLocalShards) {
|
||||
preferenceRules.add(new PreferenceRule(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION, ShardParams.REPLICA_LOCAL));
|
||||
}
|
||||
|
||||
NodePreferenceRulesComparator replicaComp = new NodePreferenceRulesComparator(preferenceRules, requestParams, nodeName, localHostAddress, sysPropsCacher, defaultRltFactory, stableRltFactory);
|
||||
ReplicaListTransformer baseReplicaListTransformer = replicaComp.getBaseReplicaListTransformer();
|
||||
if (replicaComp.getSortRules() == null) {
|
||||
// only applying base transformation
|
||||
return baseReplicaListTransformer;
|
||||
} else {
|
||||
return new TopLevelReplicaListTransformer(replicaComp, baseReplicaListTransformer);
|
||||
}
|
||||
}
|
||||
|
||||
return defaultRltFactory.getInstance(null, requestParams, RANDOM_RLTF);
|
||||
}
|
||||
|
||||
/**
|
||||
* Private class responsible for applying pairwise sort based on inherent replica attributes,
|
||||
* and subsequently reordering any equivalent replica sets according to behavior specified
|
||||
* by the baseReplicaListTransformer.
|
||||
*/
|
||||
private static final class TopLevelReplicaListTransformer implements ReplicaListTransformer {
|
||||
|
||||
private final NodePreferenceRulesComparator replicaComp;
|
||||
private final ReplicaListTransformer baseReplicaListTransformer;
|
||||
|
||||
public TopLevelReplicaListTransformer(NodePreferenceRulesComparator replicaComp, ReplicaListTransformer baseReplicaListTransformer) {
|
||||
this.replicaComp = replicaComp;
|
||||
this.baseReplicaListTransformer = baseReplicaListTransformer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void transform(List<?> choices) {
|
||||
if (choices.size() > 1) {
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Applying the following sorting preferences to replicas: {}",
|
||||
Arrays.toString(replicaComp.getPreferenceRules().toArray()));
|
||||
}
|
||||
|
||||
// First, sort according to comparator rules.
|
||||
try {
|
||||
choices.sort(replicaComp);
|
||||
} catch (IllegalArgumentException iae) {
|
||||
throw new SolrException(
|
||||
ErrorCode.BAD_REQUEST,
|
||||
iae.getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
// Next determine all boundaries between replicas ranked as "equivalent" by the comparator
|
||||
Iterator<?> iter = choices.iterator();
|
||||
Object prev = iter.next();
|
||||
Object current;
|
||||
int idx = 1;
|
||||
int boundaryCount = 0;
|
||||
int[] boundaries = new int[choices.size() - 1];
|
||||
do {
|
||||
current = iter.next();
|
||||
if (replicaComp.compare(prev, current) != 0) {
|
||||
boundaries[boundaryCount++] = idx;
|
||||
}
|
||||
prev = current;
|
||||
idx++;
|
||||
} while (iter.hasNext());
|
||||
|
||||
// Finally inspect boundaries to apply base transformation, where necessary (separate phase to avoid ConcurrentModificationException)
|
||||
int startIdx = 0;
|
||||
int endIdx;
|
||||
for (int i = 0; i < boundaryCount; i++) {
|
||||
endIdx = boundaries[i];
|
||||
if (endIdx - startIdx > 1) {
|
||||
baseReplicaListTransformer.transform(choices.subList(startIdx, endIdx));
|
||||
}
|
||||
startIdx = endIdx;
|
||||
}
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug("Applied sorting preferences to replica list: {}",
|
||||
Arrays.toString(choices.toArray()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -14,13 +14,13 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.component;
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
class ShufflingReplicaListTransformer implements ReplicaListTransformer {
|
||||
public class ShufflingReplicaListTransformer implements ReplicaListTransformer {
|
||||
|
||||
private final Random r;
|
||||
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.cloud;
|
||||
package org.apache.solr.common.cloud;
|
||||
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -29,13 +29,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
|||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
|
||||
import org.apache.solr.client.solrj.impl.PreferenceRule;
|
||||
import org.apache.solr.client.solrj.routing.PreferenceRule;
|
||||
import org.apache.solr.common.SolrCloseable;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.util.TestInjection;
|
||||
import org.apache.solr.common.util.CommonTestInjection;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
|
@ -55,7 +53,7 @@ public class NodesSysPropsCacher implements SolrCloseable {
|
|||
|
||||
private final AtomicBoolean isRunning = new AtomicBoolean(false);
|
||||
private final NodeStateProvider nodeStateProvider;
|
||||
private final Map<String, String> additionalProps = TestInjection.injectAdditionalProps();
|
||||
private Map<String, String> additionalProps = CommonTestInjection.injectAdditionalProps();
|
||||
private final String currentNode;
|
||||
private final ConcurrentHashMap<String, Map<String, Object>> cache = new ConcurrentHashMap<>();
|
||||
private final AtomicInteger fetchCounting = new AtomicInteger(0);
|
||||
|
|
@ -63,7 +61,7 @@ public class NodesSysPropsCacher implements SolrCloseable {
|
|||
private volatile boolean isClosed;
|
||||
private volatile Collection<String> tags = new ArrayList<>();
|
||||
|
||||
NodesSysPropsCacher(NodeStateProvider nodeStateProvider,
|
||||
public NodesSysPropsCacher(NodeStateProvider nodeStateProvider,
|
||||
String currentNode,
|
||||
ZkStateReader stateReader) {
|
||||
this.nodeStateProvider = nodeStateProvider;
|
||||
|
|
@ -177,12 +175,10 @@ public class NodesSysPropsCacher implements SolrCloseable {
|
|||
return result;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public int getCacheSize() {
|
||||
return cache.size();
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public boolean isRunning() {
|
||||
return isRunning.get();
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.common.util;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Allows random faults to be injected in running code during test runs across all solr packages.
|
||||
*
|
||||
* @lucene.internal
|
||||
*/
|
||||
public class CommonTestInjection {
|
||||
|
||||
private volatile static Map<String, String> additionalSystemProps = null;
|
||||
|
||||
public static void reset() {
|
||||
additionalSystemProps = null;
|
||||
}
|
||||
|
||||
public static void setAdditionalProps(Map<String, String> additionalSystemProps) {
|
||||
CommonTestInjection.additionalSystemProps = additionalSystemProps;
|
||||
}
|
||||
|
||||
public static Map<String,String> injectAdditionalProps() {
|
||||
return additionalSystemProps;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.junit.Test;
|
||||
|
||||
public class NodePreferenceRulesComparatorTest extends SolrTestCaseJ4 {
|
||||
|
||||
@Test
|
||||
public void replicaLocationTest() {
|
||||
List<Replica> replicas = getBasicReplicaList();
|
||||
|
||||
// replicaLocation rule
|
||||
List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2:8983");
|
||||
NodePreferenceRulesComparator comparator = new NodePreferenceRulesComparator(rules, null);
|
||||
replicas.sort(comparator);
|
||||
assertEquals("node2", replicas.get(0).getNodeName());
|
||||
assertEquals("node1", replicas.get(1).getNodeName());
|
||||
|
||||
}
|
||||
|
||||
public void replicaTypeTest() {
|
||||
List<Replica> replicas = getBasicReplicaList();
|
||||
|
||||
List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG");
|
||||
NodePreferenceRulesComparator comparator = new NodePreferenceRulesComparator(rules, null);
|
||||
|
||||
replicas.sort(comparator);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node2", replicas.get(1).getNodeName());
|
||||
|
||||
// reversed rule
|
||||
rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT");
|
||||
comparator = new NodePreferenceRulesComparator(rules, null);
|
||||
|
||||
replicas.sort(comparator);
|
||||
assertEquals("node2", replicas.get(0).getNodeName());
|
||||
assertEquals("node1", replicas.get(1).getNodeName());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
public void replicaTypeAndReplicaLocationTest() {
|
||||
List<Replica> replicas = getBasicReplicaList();
|
||||
// Add a replica so that sorting by replicaType:TLOG can cause a tie
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node4",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node4",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "TLOG"
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
List<PreferenceRule> rules = PreferenceRule.from(
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2_2");
|
||||
NodePreferenceRulesComparator comparator = new NodePreferenceRulesComparator(rules, null);
|
||||
|
||||
replicas.sort(comparator);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node4", replicas.get(1).getNodeName());
|
||||
assertEquals("node2", replicas.get(2).getNodeName());
|
||||
assertEquals("node3", replicas.get(3).getNodeName());
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void badRuleTest() {
|
||||
try {
|
||||
PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE);
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertEquals("Invalid shards.preference rule: " + ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE, e.getMessage());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void unknownRuleTest() {
|
||||
List<Replica> replicas = getBasicReplicaList();
|
||||
List<PreferenceRule> rules = PreferenceRule.from("badRule:test");
|
||||
try {
|
||||
replicas.sort(new NodePreferenceRulesComparator(rules, null));
|
||||
} catch (IllegalArgumentException e) {
|
||||
assertEquals("Invalid shards.preference type: badRule", e.getMessage());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<Replica> getBasicReplicaList() {
|
||||
List<Replica> replicas = new ArrayList<Replica>();
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node1",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host1:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node1",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "NRT"
|
||||
)
|
||||
)
|
||||
);
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node2",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node2",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "TLOG"
|
||||
)
|
||||
)
|
||||
);
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node3",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node3",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "PULL"
|
||||
)
|
||||
)
|
||||
);
|
||||
return replicas;
|
||||
}
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.component;
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
|
@ -28,6 +28,7 @@ import org.apache.solr.SolrTestCase;
|
|||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.handler.component.HttpShardHandlerFactory;
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.junit.Test;
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.cloud.Replica;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.junit.Test;
|
||||
|
||||
public class RequestReplicaListTransformerGeneratorTest extends SolrTestCaseJ4 {
|
||||
|
||||
@Test
|
||||
public void testNodePreferenceRulesBase() {
|
||||
RequestReplicaListTransformerGenerator generator = new RequestReplicaListTransformerGenerator();
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
List<Replica> replicas = getBasicReplicaList();
|
||||
|
||||
String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
|
||||
|
||||
params.add("routingPreference", "0");
|
||||
params.add(ShardParams.SHARDS_PREFERENCE, rulesParam);
|
||||
|
||||
ReplicaListTransformer rlt = generator.getReplicaListTransformer(params);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node2", replicas.get(1).getNodeName());
|
||||
assertEquals("node3", replicas.get(2).getNodeName());
|
||||
|
||||
params.set("routingPreference", "1");
|
||||
rlt = generator.getReplicaListTransformer(params);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node2", replicas.get(0).getNodeName());
|
||||
assertEquals("node3", replicas.get(1).getNodeName());
|
||||
assertEquals("node1", replicas.get(2).getNodeName());
|
||||
|
||||
params.set("routingPreference", "2");
|
||||
rlt = generator.getReplicaListTransformer(params);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node3", replicas.get(0).getNodeName());
|
||||
assertEquals("node1", replicas.get(1).getNodeName());
|
||||
assertEquals("node2", replicas.get(2).getNodeName());
|
||||
|
||||
params.set("routingPreference", "3");
|
||||
rlt = generator.getReplicaListTransformer(params);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node2", replicas.get(1).getNodeName());
|
||||
assertEquals("node3", replicas.get(2).getNodeName());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
public void replicaTypeAndReplicaBase() {
|
||||
RequestReplicaListTransformerGenerator generator = new RequestReplicaListTransformerGenerator();
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
List<Replica> replicas = getBasicReplicaList();
|
||||
|
||||
// Add a replica so that sorting by replicaType:TLOG can cause a tie
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node4",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node4",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "TLOG"
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// replicaType and replicaBase combined rule param
|
||||
String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
|
||||
ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
|
||||
|
||||
params.add("routingPreference", "0");
|
||||
params.add(ShardParams.SHARDS_PREFERENCE, rulesParam);
|
||||
ReplicaListTransformer rlt = generator.getReplicaListTransformer(params);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node2", replicas.get(1).getNodeName());
|
||||
assertEquals("node4", replicas.get(2).getNodeName());
|
||||
assertEquals("node3", replicas.get(3).getNodeName());
|
||||
|
||||
params.set("routingPreference", "1");
|
||||
rlt = generator.getReplicaListTransformer(params);
|
||||
rlt.transform(replicas);
|
||||
assertEquals("node1", replicas.get(0).getNodeName());
|
||||
assertEquals("node4", replicas.get(1).getNodeName());
|
||||
assertEquals("node2", replicas.get(2).getNodeName());
|
||||
assertEquals("node3", replicas.get(3).getNodeName());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<Replica> getBasicReplicaList() {
|
||||
List<Replica> replicas = new ArrayList<Replica>();
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node1",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host1:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node1",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "NRT"
|
||||
)
|
||||
)
|
||||
);
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node2",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node2",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "TLOG"
|
||||
)
|
||||
)
|
||||
);
|
||||
replicas.add(
|
||||
new Replica(
|
||||
"node3",
|
||||
map(
|
||||
ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
|
||||
ZkStateReader.NODE_NAME_PROP, "node3",
|
||||
ZkStateReader.CORE_NAME_PROP, "collection1",
|
||||
ZkStateReader.REPLICA_TYPE, "PULL"
|
||||
)
|
||||
)
|
||||
);
|
||||
return replicas;
|
||||
}
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.component;
|
||||
package org.apache.solr.client.solrj.routing;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
Loading…
Reference in New Issue