Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-11444: Improve consistency of collection alias handling and collection list references. 

Other refactorings of nearby code too.
This commit is contained in:
David Smiley 2017-10-19 00:02:24 -04:00
parent 99e853faf8
commit e001f35289
25 changed files with 526 additions and 617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
solr/CHANGES.txt
View File

@ -79,6 +79,11 @@ Other Changes
* SOLR-11464, SOLR-11493: Minor refactorings to DistributedUpdateProcessor. (Gus Heck, David Smiley) * SOLR-11464, SOLR-11493: Minor refactorings to DistributedUpdateProcessor. (Gus Heck, David Smiley)
* SOLR-11444: Improved consistency of collection alias handling, and collection references that are
comma delimited lists of collections, across the various places collections can be referred to.
Updates are delivered to the first in a list. It's now possible to refer to a comma delimited list
of them in the path, ex: /solr/colA,colB/select?... (David Smiley)
================== 7.1.0 ================== ================== 7.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

36
solr/core/src/java/org/apache/solr/api/V2HttpCall.java
View File

@ -53,6 +53,7 @@ import org.apache.solr.common.util.PathTrie;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.params.CommonParams.JSON; import static org.apache.solr.common.params.CommonParams.JSON;
import static org.apache.solr.common.params.CommonParams.WT; import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN; import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN;
@ -105,28 +106,30 @@ public class V2HttpCall extends HttpSolrCall {
} }
if ("c".equals(prefix) || "collections".equals(prefix)) { if ("c".equals(prefix) || "collections".equals(prefix)) {
String collectionName = origCorename = corename = pieces.get(1); origCorename = pieces.get(1);
collectionsList = resolveCollectionListOrAlias(queryParams.get(COLLECTION_PROP, origCorename));
String collectionName = collectionsList.get(0); // first
//TODO try the other collections if can't find a local replica of the first?
DocCollection collection = getDocCollection(collectionName); DocCollection collection = getDocCollection(collectionName);
if (collection == null) { if (collection == null) {
if ( ! path.endsWith(CommonParams.INTROSPECT)) { if ( ! path.endsWith(CommonParams.INTROSPECT)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such collection or alias"); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such collection or alias");
} }
} else { } else {
boolean isPreferLeader = false; boolean isPreferLeader = (path.endsWith("/update") || path.contains("/update/"));
if (path.endsWith("/update") || path.contains("/update/")) {
isPreferLeader = true;
}
core = getCoreByCollection(collection.getName(), isPreferLeader); core = getCoreByCollection(collection.getName(), isPreferLeader);
if (core == null) { if (core == null) {
//this collection exists , but this node does not have a replica for that collection //this collection exists , but this node does not have a replica for that collection
//todo find a better way to compute remote //todo find a better way to compute remote
extractRemotePath(corename, origCorename, 0); extractRemotePath(collectionName, origCorename, 0);
return; return;
} }
} }
} else if ("cores".equals(prefix)) { } else if ("cores".equals(prefix)) {
origCorename = corename = pieces.get(1); origCorename = pieces.get(1);
core = cores.getCore(corename); core = cores.getCore(origCorename);
} }
if (core == null) { if (core == null) {
log.error(">> path: '" + path + "'"); log.error(">> path: '" + path + "'");
@ -134,7 +137,7 @@ public class V2HttpCall extends HttpSolrCall {
initAdminRequest(path); initAdminRequest(path);
return; return;
} else { } else {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no core retrieved for " + corename); throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no core retrieved for " + origCorename);
} }
} }
@ -148,9 +151,7 @@ public class V2HttpCall extends HttpSolrCall {
MDCLoggingContext.setCore(core); MDCLoggingContext.setCore(core);
parseRequest(); parseRequest();
if (usingAliases) { addCollectionParamIfNeeded(getCollectionsList());
processAliases(aliases, collectionsList);
}
action = PROCESS; action = PROCESS;
// we are done with a valid handler // we are done with a valid handler
@ -180,17 +181,12 @@ public class V2HttpCall extends HttpSolrCall {
if (solrReq == null) solrReq = parser.parse(core, path, req); if (solrReq == null) solrReq = parser.parse(core, path, req);
} }
protected DocCollection getDocCollection(String collectionName) { protected DocCollection getDocCollection(String collectionName) { // note: don't send an alias; resolve it first
if (!cores.isZooKeeperAware()) { if (!cores.isZooKeeperAware()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Solr not running in cloud mode "); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Solr not running in cloud mode ");
} }
ZkStateReader zkStateReader = cores.getZkController().getZkStateReader(); ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
DocCollection collection = zkStateReader.getClusterState().getCollectionOrNull(collectionName); return zkStateReader.getClusterState().getCollectionOrNull(collectionName);
if (collection == null) {
collectionName = corename = lookupAliases(collectionName);
collection = zkStateReader.getClusterState().getCollectionOrNull(collectionName);
}
return collection;
} }
public static Api getApiInfo(PluginBag<SolrRequestHandler> requestHandlers, public static Api getApiInfo(PluginBag<SolrRequestHandler> requestHandlers,

52
solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java
View File

@ -17,27 +17,28 @@
*/ */
package org.apache.solr.cloud; package org.apache.solr.cloud;
import static org.apache.solr.common.params.CommonParams.NAME;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.util.HashMap; import java.util.HashSet;
import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.StrUtils;
import org.apache.solr.util.TimeOut; import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.NAME;
public class CreateAliasCmd implements Cmd { public class CreateAliasCmd implements Cmd {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -51,24 +52,12 @@ public class CreateAliasCmd implements Cmd {
public void call(ClusterState state, ZkNodeProps message, NamedList results) public void call(ClusterState state, ZkNodeProps message, NamedList results)
throws Exception { throws Exception {
String aliasName = message.getStr(NAME); String aliasName = message.getStr(NAME);
String collections = message.getStr("collections"); String collections = message.getStr("collections"); // could be comma delimited list
ZkStateReader zkStateReader = ocmh.zkStateReader; ZkStateReader zkStateReader = ocmh.zkStateReader;
Map<String, String> prevColAliases = zkStateReader.getAliases().getCollectionAliasMap(); validateAllCollectionsExistAndNoDups(collections, zkStateReader);
validateAllCollectionsExist(collections, prevColAliases, zkStateReader.getClusterState());
Map<String, Map<String, String>> newAliasesMap = new HashMap<>(); byte[] jsonBytes = zkStateReader.getAliases().cloneWithCollectionAlias(aliasName, collections).toJSON();
Map<String, String> newCollectionAliasesMap = new HashMap<>();
if (prevColAliases != null) {
newCollectionAliasesMap.putAll(prevColAliases);
}
newCollectionAliasesMap.put(aliasName, collections);
newAliasesMap.put("collection", newCollectionAliasesMap);
Aliases newAliases = new Aliases(newAliasesMap);
byte[] jsonBytes = null;
if (newAliases.collectionAliasSize() > 0) { // only sub map right now
jsonBytes = Utils.toJSON(newAliases.getAliasMap());
}
try { try {
zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true); zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true);
@ -84,10 +73,16 @@ public class CreateAliasCmd implements Cmd {
} }
} }
private void validateAllCollectionsExist(String collections, Map<String,String> prevColAliases, ClusterState clusterState) { private void validateAllCollectionsExistAndNoDups(String collections, ZkStateReader zkStateReader) {
String[] collectionArr = collections.split(","); List<String> collectionArr = StrUtils.splitSmart(collections, ",", true);
for (String collection:collectionArr) { if (new HashSet<>(collectionArr).size() != collectionArr.size()) {
if (clusterState.getCollectionOrNull(collection) == null && (prevColAliases == null || !prevColAliases.containsKey(collection))) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
String.format(Locale.ROOT, "Can't create collection alias for collections='%s', since it contains duplicates", collections));
}
ClusterState clusterState = zkStateReader.getClusterState();
Set<String> aliasNames = zkStateReader.getAliases().getCollectionAliasListMap().keySet();
for (String collection : collectionArr) {
if (clusterState.getCollectionOrNull(collection) == null && !aliasNames.contains(collection)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
String.format(Locale.ROOT, "Can't create collection alias for collections='%s', '%s' is not an existing collection or alias", collections, collection)); String.format(Locale.ROOT, "Can't create collection alias for collections='%s', '%s' is not an existing collection or alias", collections, collection));
} }
@ -95,14 +90,11 @@ public class CreateAliasCmd implements Cmd {
} }
private void checkForAlias(String name, String value) { private void checkForAlias(String name, String value) {
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
boolean success = false; boolean success = false;
Aliases aliases;
while (!timeout.hasTimedOut()) { while (!timeout.hasTimedOut()) {
aliases = ocmh.zkStateReader.getAliases(); String collections = ocmh.zkStateReader.getAliases().getCollectionAliasMap().get(name);
String collections = aliases.getCollectionAlias(name); if (Objects.equals(collections, value)) {
if (collections != null && collections.equals(value)) {
success = true; success = true;
break; break;
} }

24
solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java
View File

@ -19,17 +19,13 @@
package org.apache.solr.cloud; package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.TimeOut; import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -49,20 +45,11 @@ public class DeleteAliasCmd implements OverseerCollectionMessageHandler.Cmd {
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
String aliasName = message.getStr(NAME); String aliasName = message.getStr(NAME);
Map<String,Map<String,String>> newAliasesMap = new HashMap<>();
Map<String,String> newCollectionAliasesMap = new HashMap<>();
ZkStateReader zkStateReader = ocmh.zkStateReader; ZkStateReader zkStateReader = ocmh.zkStateReader;
newCollectionAliasesMap.putAll(zkStateReader.getAliases().getCollectionAliasMap()); byte[] jsonBytes = zkStateReader.getAliases().cloneWithCollectionAlias(aliasName, null).toJSON();
newCollectionAliasesMap.remove(aliasName);
newAliasesMap.put("collection", newCollectionAliasesMap);
Aliases newAliases = new Aliases(newAliasesMap);
byte[] jsonBytes = null;
if (newAliases.collectionAliasSize() > 0) { // only sub map right now
jsonBytes = Utils.toJSON(newAliases.getAliasMap());
}
try { try {
zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true);
jsonBytes, true);
checkForAliasAbsence(aliasName); checkForAliasAbsence(aliasName);
// some fudge for other nodes // some fudge for other nodes
Thread.sleep(100); Thread.sleep(100);
@ -76,13 +63,10 @@ public class DeleteAliasCmd implements OverseerCollectionMessageHandler.Cmd {
} }
private void checkForAliasAbsence(String name) { private void checkForAliasAbsence(String name) {
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS); TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
boolean success = false; boolean success = false;
Aliases aliases = null;
while (! timeout.hasTimedOut()) { while (! timeout.hasTimedOut()) {
aliases = ocmh.zkStateReader.getAliases(); String collections = ocmh.zkStateReader.getAliases().getCollectionAliasMap().get(name);
String collections = aliases.getCollectionAlias(name);
if (collections == null) { if (collections == null) {
success = true; success = true;
break; break;

34
solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
View File

@ -38,20 +38,18 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.Utils;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
public class ClusterStatus { public class ClusterStatus {
private final ZkStateReader zkStateReader; private final ZkStateReader zkStateReader;
private final String collection; private final ZkNodeProps message;
private ZkNodeProps message; private final String collection; // maybe null
public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) { public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) {
this.zkStateReader = zkStateReader; this.zkStateReader = zkStateReader;
this.message = props; this.message = props;
collection = props.getStr(ZkStateReader.COLLECTION_PROP); collection = props.getStr(ZkStateReader.COLLECTION_PROP);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -60,20 +58,14 @@ public class ClusterStatus {
// read aliases // read aliases
Aliases aliases = zkStateReader.getAliases(); Aliases aliases = zkStateReader.getAliases();
Map<String, List<String>> collectionVsAliases = new HashMap<>(); Map<String, List<String>> collectionVsAliases = new HashMap<>();
Map<String, String> aliasVsCollections = aliases.getCollectionAliasMap(); Map<String, List<String>> aliasVsCollections = aliases.getCollectionAliasListMap();
if (aliasVsCollections != null) { for (Map.Entry<String, List<String>> entry : aliasVsCollections.entrySet()) {
for (Map.Entry<String, String> entry : aliasVsCollections.entrySet()) { String alias = entry.getKey();
List<String> colls = StrUtils.splitSmart(entry.getValue(), ','); List<String> colls = entry.getValue();
String alias = entry.getKey(); for (String coll : colls) {
for (String coll : colls) { if (collection == null || collection.equals(coll)) {
if (collection == null || collection.equals(coll)) { List<String> list = collectionVsAliases.computeIfAbsent(coll, k -> new ArrayList<>());
List<String> list = collectionVsAliases.get(coll); list.add(alias);
if (list == null) {
list = new ArrayList<>();
collectionVsAliases.put(coll, list);
}
list.add(alias);
}
} }
} }
} }
@ -158,8 +150,9 @@ public class ClusterStatus {
} }
// add the alias map too // add the alias map too
if (aliasVsCollections != null && !aliasVsCollections.isEmpty()) { Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap(); // comma delim
clusterStatus.add("aliases", aliasVsCollections); if (!collectionAliasMap.isEmpty()) {
clusterStatus.add("aliases", collectionAliasMap);
} }
// add the roles map // add the roles map
@ -172,6 +165,7 @@ public class ClusterStatus {
results.add("cluster", clusterStatus); results.add("cluster", clusterStatus);
} }
/** /**
* Get collection status from cluster state. * Get collection status from cluster state.
* Can return collection status by given shard name. * Can return collection status by given shard name.

6
solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
View File

@ -58,10 +58,8 @@ class SolrSchema extends AbstractSchema {
} }
Aliases aliases = zkStateReader.getAliases(); Aliases aliases = zkStateReader.getAliases();
if(aliases.collectionAliasSize() > 0) { for (String alias : aliases.getCollectionAliasListMap().keySet()) {
for (Map.Entry<String, String> alias : aliases.getCollectionAliasMap().entrySet()) { builder.put(alias, new SolrTable(this, alias));
builder.put(alias.getKey(), new SolrTable(this, alias.getValue()));
}
} }
return builder.build(); return builder.build();

30
solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
View File

@ -17,7 +17,7 @@
package org.apache.solr.search.join; package org.apache.solr.search.join;
import java.io.IOException; import java.io.IOException;
import java.util.Map; import java.util.List;
import java.util.Objects; import java.util.Objects;
import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocValuesType;
@ -275,10 +275,9 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
public static String getCoreName(final String fromIndex, CoreContainer container) { public static String getCoreName(final String fromIndex, CoreContainer container) {
if (container.isZooKeeperAware()) { if (container.isZooKeeperAware()) {
ZkController zkController = container.getZkController(); ZkController zkController = container.getZkController();
final String resolved = final String resolved = resolveAlias(fromIndex, zkController);
zkController.getClusterState().hasCollection(fromIndex) // TODO DWS: no need for this since later, clusterState.getCollection will throw a reasonable error
? fromIndex : resolveAlias(fromIndex, zkController); if (!zkController.getClusterState().hasCollection(resolved)) {
if (resolved == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"SolrCloud join: Collection '" + fromIndex + "' not found!"); "SolrCloud join: Collection '" + fromIndex + "' not found!");
} }
@ -289,21 +288,14 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
private static String resolveAlias(String fromIndex, ZkController zkController) { private static String resolveAlias(String fromIndex, ZkController zkController) {
final Aliases aliases = zkController.getZkStateReader().getAliases(); final Aliases aliases = zkController.getZkStateReader().getAliases();
if (aliases != null) { List<String> collections = aliases.resolveAliases(fromIndex); // if not an alias, returns input
final String resolved; if (collections.size() != 1) {
Map<String, String> collectionAliases = aliases.getCollectionAliasMap(); throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
resolved = (collectionAliases != null) ? collectionAliases.get(fromIndex) : null; "SolrCloud join: Collection alias '" + fromIndex +
if (resolved != null) { "' maps to multiple collections (" + collections +
if (resolved.split(",").length > 1) { "), which is not currently supported for joins.");
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"SolrCloud join: Collection alias '" + fromIndex +
"' maps to multiple collections (" + resolved +
"), which is not currently supported for joins.");
}
return resolved;
}
} }
return null; return collections.get(0);
} }
private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) { private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) {

248
solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
View File

@ -30,8 +30,8 @@ import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Random; import java.util.Random;
@ -72,7 +72,9 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.JsonSchemaValidator;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.StrUtils;
@ -101,8 +103,6 @@ import org.apache.solr.servlet.SolrDispatchFilter.Action;
import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
import org.apache.solr.servlet.cache.Method; import org.apache.solr.servlet.cache.Method;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory; import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.JsonSchemaValidator;
import org.apache.solr.util.RTimerTree; import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.TimeOut; import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
@ -160,13 +160,10 @@ public class HttpSolrCall {
protected String coreUrl; protected String coreUrl;
protected SolrConfig config; protected SolrConfig config;
protected Map<String, Integer> invalidStates; protected Map<String, Integer> invalidStates;
protected boolean usingAliases = false;
//The states of client that is invalid in this request //The states of client that is invalid in this request
protected Aliases aliases = null; protected String origCorename; // What's in the URL path; might reference a collection/alias or a Solr core name
protected String corename = ""; protected List<String> collectionsList; // The list of SolrCloud collections if in SolrCloud (usually 1)
protected String origCorename = null;
public RequestType getRequestType() { public RequestType getRequestType() {
return requestType; return requestType;
@ -174,13 +171,6 @@ public class HttpSolrCall {
protected RequestType requestType; protected RequestType requestType;
public List<String> getCollectionsList() {
return collectionsList;
}
protected List<String> collectionsList;
public HttpSolrCall(SolrDispatchFilter solrDispatchFilter, CoreContainer cores, public HttpSolrCall(SolrDispatchFilter solrDispatchFilter, CoreContainer cores,
HttpServletRequest request, HttpServletResponse response, boolean retry) { HttpServletRequest request, HttpServletResponse response, boolean retry) {
this.solrDispatchFilter = solrDispatchFilter; this.solrDispatchFilter = solrDispatchFilter;
@ -206,7 +196,6 @@ public class HttpSolrCall {
return path; return path;
} }
public HttpServletRequest getReq() { public HttpServletRequest getReq() {
return req; return req;
} }
@ -219,12 +208,22 @@ public class HttpSolrCall {
return queryParams; return queryParams;
} }
protected Aliases getAliases() {
return cores.isZooKeeperAware() ? cores.getZkController().getZkStateReader().getAliases() : Aliases.EMPTY;
}
/** The collection(s) referenced in this request. Populated in {@link #init()}. Not null. */
public List<String> getCollectionsList() {
return collectionsList != null ? collectionsList : Collections.emptyList();
}
protected void init() throws Exception { protected void init() throws Exception {
// check for management path // check for management path
String alternate = cores.getManagementPath(); String alternate = cores.getManagementPath();
if (alternate != null && path.startsWith(alternate)) { if (alternate != null && path.startsWith(alternate)) {
path = path.substring(0, alternate.length()); path = path.substring(0, alternate.length());
} }
// unused feature ? // unused feature ?
int idx = path.indexOf(':'); int idx = path.indexOf(':');
if (idx > 0) { if (idx > 0) {
@ -232,8 +231,6 @@ public class HttpSolrCall {
path = path.substring(0, idx); path = path.substring(0, idx);
} }
boolean usingAliases = false;
// Check for container handlers // Check for container handlers
handler = cores.getRequestHandler(path); handler = cores.getRequestHandler(path);
if (handler != null) { if (handler != null) {
@ -242,69 +239,59 @@ public class HttpSolrCall {
requestType = RequestType.ADMIN; requestType = RequestType.ADMIN;
action = ADMIN; action = ADMIN;
return; return;
} else { }
//otherwise, we should find a core from the path
idx = path.indexOf("/", 1);
if (idx > 1) {
// try to get the corename as a request parameter first
corename = path.substring(1, idx);
// look at aliases // Parse a core or collection name from the path and attempt to see if it's a core name
if (cores.isZooKeeperAware()) { idx = path.indexOf("/", 1);
origCorename = corename; if (idx > 1) {
ZkStateReader reader = cores.getZkController().getZkStateReader(); origCorename = path.substring(1, idx);
aliases = reader.getAliases();
if (aliases != null && aliases.collectionAliasSize() > 0) { // Try to resolve a Solr core name
usingAliases = true; core = cores.getCore(origCorename);
String alias = aliases.getCollectionAlias(corename); if (core != null) {
if (alias != null) { path = path.substring(idx);
collectionsList = StrUtils.splitSmart(alias, ",", true); } else {
corename = collectionsList.get(0); if (cores.isCoreLoading(origCorename)) { // extra mem barriers, so don't look at this before trying to get core
} throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
}
} }
// the core may have just finished loading
core = cores.getCore(corename); core = cores.getCore(origCorename);
if (core != null) { if (core != null) {
path = path.substring(idx); path = path.substring(idx);
} else if (cores.isCoreLoading(corename)) { // extra mem barriers, so don't look at this before trying to get core
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
} else { } else {
// the core may have just finished loading if (!cores.isZooKeeperAware()) {
core = cores.getCore(corename); core = cores.getCore("");
if (core != null) {
path = path.substring(idx);
} }
} }
} }
if (core == null) {
if (!cores.isZooKeeperAware()) {
core = cores.getCore("");
}
}
} }
if (core == null && cores.isZooKeeperAware()) { if (cores.isZooKeeperAware()) {
// we couldn't find the core - lets make sure a collection was not specified instead // init collectionList (usually one name but not when there are aliases)
boolean isPreferLeader = false; String def = core != null ? core.getCoreDescriptor().getCollectionName() : origCorename;
if (path.endsWith("/update") || path.contains("/update/")) { collectionsList = resolveCollectionListOrAlias(queryParams.get(COLLECTION_PROP, def)); // &collection= takes precedence
isPreferLeader = true;
}
core = getCoreByCollection(corename, isPreferLeader);
if (core != null) {
// we found a core, update the path
path = path.substring(idx);
if (collectionsList == null)
collectionsList = new ArrayList<>();
collectionsList.add(corename);
}
// if we couldn't find it locally, look on other nodes if (core == null) {
extractRemotePath(corename, origCorename, idx); // lookup core from collection, or route away if need to
if (action != null) return; String collectionName = collectionsList.isEmpty() ? null : collectionsList.get(0); // route to 1st
//core is not available locally or remotely //TODO try the other collections if can't find a local replica of the first? (and do to V2HttpSolrCall)
autoCreateSystemColl(corename);
if(action != null) return; boolean isPreferLeader = (path.endsWith("/update") || path.contains("/update/"));
core = getCoreByCollection(collectionName, isPreferLeader); // find a local replica/core for the collection
if (core != null) {
if (idx > 0) {
path = path.substring(idx);
}
} else {
// if we couldn't find it locally, look on other nodes
extractRemotePath(collectionName, origCorename, idx);
if (action != null) return;
//core is not available locally or remotely
autoCreateSystemColl(collectionName);
if (action != null) return;
}
}
} }
// With a valid core... // With a valid core...
@ -314,7 +301,6 @@ public class HttpSolrCall {
// get or create/cache the parser for the core // get or create/cache the parser for the core
SolrRequestParsers parser = config.getRequestParsers(); SolrRequestParsers parser = config.getRequestParsers();
// Determine the handler from the url path if not set // Determine the handler from the url path if not set
// (we might already have selected the cores handler) // (we might already have selected the cores handler)
extractHandlerFromURLPath(parser); extractHandlerFromURLPath(parser);
@ -329,9 +315,7 @@ public class HttpSolrCall {
invalidStates = checkStateVersionsAreValid(solrReq.getParams().get(CloudSolrClient.STATE_VERSION)); invalidStates = checkStateVersionsAreValid(solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
if (usingAliases) { addCollectionParamIfNeeded(getCollectionsList());
processAliases(aliases, collectionsList);
}
action = PROCESS; action = PROCESS;
return; // we are done with a valid handler return; // we are done with a valid handler
@ -374,18 +358,24 @@ public class HttpSolrCall {
} }
} }
protected String lookupAliases(String collName) { /**
ZkStateReader reader = cores.getZkController().getZkStateReader(); * Resolves the parameter as a potential comma delimited list of collections, and resolves aliases too.
aliases = reader.getAliases(); * One level of aliases pointing to another alias is supported.
if (aliases != null && aliases.collectionAliasSize() > 0) { * De-duplicates and retains the order.
usingAliases = true; * {@link #getCollectionsList()}
String alias = aliases.getCollectionAlias(collName); */
if (alias != null) { protected List<String> resolveCollectionListOrAlias(String collectionStr) {
collectionsList = StrUtils.splitSmart(alias, ",", true); if (collectionStr == null) {
return collectionsList.get(0); return Collections.emptyList();
}
} }
return null; LinkedHashSet<String> resultList = new LinkedHashSet<>();
Aliases aliases = getAliases();
List<String> inputCollections = StrUtils.splitSmart(collectionStr, ",", true);
for (String inputCollection : inputCollections) {
List<String> resolvedCollections = aliases.resolveAliases(inputCollection);
resultList.addAll(resolvedCollections);
}
return new ArrayList<>(resultList);
} }
/** /**
@ -432,9 +422,9 @@ public class HttpSolrCall {
} }
} }
protected void extractRemotePath(String corename, String origCorename, int idx) throws UnsupportedEncodingException, KeeperException, InterruptedException { protected void extractRemotePath(String collectionName, String origCorename, int idx) throws UnsupportedEncodingException, KeeperException, InterruptedException {
if (core == null && idx > 0) { if (core == null && idx > 0) {
coreUrl = getRemotCoreUrl(corename, origCorename); coreUrl = getRemotCoreUrl(collectionName, origCorename);
// don't proxy for internal update requests // don't proxy for internal update requests
invalidStates = checkStateVersionsAreValid(queryParams.get(CloudSolrClient.STATE_VERSION)); invalidStates = checkStateVersionsAreValid(queryParams.get(CloudSolrClient.STATE_VERSION));
if (coreUrl != null if (coreUrl != null
@ -745,40 +735,32 @@ public class HttpSolrCall {
handler.handleRequest(solrReq, solrResp); handler.handleRequest(solrReq, solrResp);
} }
protected void processAliases(Aliases aliases, /**
List<String> collectionsList) { * Sets the "collection" parameter on the request to the list of alias-resolved collections for this request.
String collection = solrReq.getParams().get(COLLECTION_PROP); * It can be avoided sometimes.
if (collection != null) { * Note: {@link org.apache.solr.handler.component.HttpShardHandler} processes this param.
collectionsList = StrUtils.splitSmart(collection, ",", true); * @see #getCollectionsList()
*/
protected void addCollectionParamIfNeeded(List<String> collections) {
if (collections.isEmpty()) {
return;
} }
if (collectionsList != null) { assert cores.isZooKeeperAware();
Set<String> newCollectionsList = new HashSet<>( String collectionParam = queryParams.get(COLLECTION_PROP);
collectionsList.size()); // if there is no existing collection param and the core we go to is for the expected collection,
for (String col : collectionsList) { // then we needn't add a collection param
String al = aliases.getCollectionAlias(col); if (collectionParam == null && // if collection param already exists, we may need to over-write it
if (al != null) { core != null && collections.equals(Collections.singletonList(core.getCoreDescriptor().getCollectionName()))) {
List<String> aliasList = StrUtils.splitSmart(al, ",", true); return;
newCollectionsList.addAll(aliasList);
} else {
newCollectionsList.add(col);
}
}
if (newCollectionsList.size() > 0) {
StringBuilder collectionString = new StringBuilder();
Iterator<String> it = newCollectionsList.iterator();
int sz = newCollectionsList.size();
for (int i = 0; i < sz; i++) {
collectionString.append(it.next());
if (i < newCollectionsList.size() - 1) {
collectionString.append(",");
}
}
ModifiableSolrParams params = new ModifiableSolrParams(
solrReq.getParams());
params.set(COLLECTION_PROP, collectionString.toString());
solrReq.setParams(params);
}
} }
String newCollectionParam = StrUtils.join(collections, ',');
if (newCollectionParam.equals(collectionParam)) {
return;
}
// TODO add a SolrRequest.getModifiableParams ?
ModifiableSolrParams params = new ModifiableSolrParams(solrReq.getParams());
params.set(COLLECTION_PROP, newCollectionParam);
solrReq.setParams(params);
} }
private void writeResponse(SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, Method reqMethod) private void writeResponse(SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, Method reqMethod)
@ -916,9 +898,6 @@ public class HttpSolrCall {
return null; return null;
} }
if (collectionsList == null)
collectionsList = new ArrayList<>();
collectionsList.add(collectionName); collectionsList.add(collectionName);
String coreUrl = getCoreUrl(collectionName, origCorename, clusterState, String coreUrl = getCoreUrl(collectionName, origCorename, clusterState,
slices, byCoreName, true); slices, byCoreName, true);
@ -984,10 +963,8 @@ public class HttpSolrCall {
SolrParams params = getQueryParams(); SolrParams params = getQueryParams();
final ArrayList<CollectionRequest> collectionRequests = new ArrayList<>(); final ArrayList<CollectionRequest> collectionRequests = new ArrayList<>();
if (getCollectionsList() != null) { for (String collection : getCollectionsList()) {
for (String collection : getCollectionsList()) { collectionRequests.add(new CollectionRequest(collection));
collectionRequests.add(new CollectionRequest(collection));
}
} }
// Extract collection name from the params in case of a Collection Admin request // Extract collection name from the params in case of a Collection Admin request
@ -999,15 +976,7 @@ public class HttpSolrCall {
else if (params.get(COLLECTION_PROP) != null) else if (params.get(COLLECTION_PROP) != null)
collectionRequests.add(new CollectionRequest(params.get(COLLECTION_PROP))); collectionRequests.add(new CollectionRequest(params.get(COLLECTION_PROP)));
} }
// Handle the case when it's a /select request and collections are specified as a param
if(resource.equals("/select") && params.get("collection") != null) {
collectionRequests.clear();
for(String collection:params.get("collection").split(",")) {
collectionRequests.add(new CollectionRequest(collection));
}
}
// Populate the request type if the request is select or update // Populate the request type if the request is select or update
if(requestType == RequestType.UNKNOWN) { if(requestType == RequestType.UNKNOWN) {
if(resource.startsWith("/select") || resource.startsWith("/get")) if(resource.startsWith("/select") || resource.startsWith("/get"))
@ -1016,15 +985,6 @@ public class HttpSolrCall {
requestType = RequestType.WRITE; requestType = RequestType.WRITE;
} }
// There's no collection explicitly mentioned, let's try and extract it from the core if one exists for
// the purpose of processing this request.
if (getCore() != null && (getCollectionsList() == null || getCollectionsList().size() == 0)) {
collectionRequests.add(new CollectionRequest(getCore().getCoreDescriptor().getCollectionName()));
}
if (getQueryParams().get(COLLECTION_PROP) != null)
collectionRequests.add(new CollectionRequest(getQueryParams().get(COLLECTION_PROP)));
return new AuthorizationContext() { return new AuthorizationContext() {
@Override @Override
public SolrParams getParams() { public SolrParams getParams() {

146
solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
View File

@ -16,7 +16,11 @@
*/ */
package org.apache.solr.cloud; package org.apache.solr.cloud;
import java.io.IOException;
import java.util.function.Consumer;
import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient;
@ -24,6 +28,8 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
@ -62,92 +68,50 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
new CollectionAdminRequest.ListAliases().process(cluster.getSolrClient()).getAliases().get("testalias")); new CollectionAdminRequest.ListAliases().process(cluster.getSolrClient()).getAliases().get("testalias"));
// search for alias // search for alias
QueryResponse res = cluster.getSolrClient().query("testalias", new SolrQuery("*:*")); searchSeveralWays("testalias", new SolrQuery("*:*"), 3);
assertEquals(3, res.getResults().getNumFound());
// Use a comma delimited list, one of which is an alias
// search for alias with random non cloud client searchSeveralWays("testalias,collection2", new SolrQuery("*:*"), 5);
JettySolrRunner jetty = cluster.getRandomJetty(random());
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
res = client.query(new SolrQuery("*:*"));
assertEquals(3, res.getResults().getNumFound());
}
// create alias, collection2 first because it's not on every node // create alias, collection2 first because it's not on every node
CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient()); CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient());
// search with new cloud client
try (CloudSolrClient cloudSolrClient = getCloudSolrClient(cluster.getZkServer().getZkAddress(), random().nextBoolean())) {
cloudSolrClient.setParallelUpdates(random().nextBoolean());
SolrQuery query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = cloudSolrClient.query(query);
assertEquals(5, res.getResults().getNumFound());
// Try with setDefaultCollection searchSeveralWays("testalias", new SolrQuery("*:*"), 5);
query = new SolrQuery("*:*");
cloudSolrClient.setDefaultCollection("testalias");
res = cloudSolrClient.query(query);
assertEquals(5, res.getResults().getNumFound());
}
// search for alias with random non cloud client
jetty = cluster.getRandomJetty(random());
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
SolrQuery query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = client.query(query);
assertEquals(5, res.getResults().getNumFound());
// now without collections param
query = new SolrQuery("*:*");
res = client.query(query);
assertEquals(5, res.getResults().getNumFound());
}
// update alias // update alias
CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient()); CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient());
// search for alias searchSeveralWays("testalias", new SolrQuery("*:*"), 2);
SolrQuery query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = cluster.getSolrClient().query(query);
assertEquals(2, res.getResults().getNumFound());
// set alias to two collections // set alias to two collections
CollectionAdminRequest.createAlias("testalias", "collection1,collection2").process(cluster.getSolrClient()); CollectionAdminRequest.createAlias("testalias", "collection1,collection2").process(cluster.getSolrClient());
searchSeveralWays("testalias", new SolrQuery("*:*"), 5);
query = new SolrQuery("*:*"); // alias pointing to alias (one level of indirection is supported; more than that is not (may or may not work)
query.set("collection", "testalias"); // TODO dubious; remove?
res = cluster.getSolrClient().query(query); CollectionAdminRequest.createAlias("testalias2", "testalias").process(cluster.getSolrClient());
assertEquals(5, res.getResults().getNumFound()); searchSeveralWays("testalias2", new SolrQuery("*:*"), 5);
// try a std client
// search 1 and 2, but have no collections param
query = new SolrQuery("*:*");
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
res = client.query(query);
assertEquals(5, res.getResults().getNumFound());
}
// Test 2 aliases pointing to the same collection
CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient()); CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient());
// a second alias
CollectionAdminRequest.createAlias("testalias2", "collection2").process(cluster.getSolrClient()); CollectionAdminRequest.createAlias("testalias2", "collection2").process(cluster.getSolrClient());
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) { // add one document to testalias, thus to collection2
new UpdateRequest() new UpdateRequest()
.add("id", "11", "a_t", "humpty dumpy4 sat on a walls") .add("id", "11", "a_t", "humpty dumpy4 sat on a walls")
.commit(cluster.getSolrClient(), "testalias"); .commit(cluster.getSolrClient(), "testalias"); // thus gets added to collection2
res = client.query(query);
assertEquals(3, res.getResults().getNumFound()); searchSeveralWays("testalias", new SolrQuery("*:*"), 3);
}
CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient()); CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient());
query = new SolrQuery("*:*"); searchSeveralWays("testalias", new SolrQuery("*:*"), 6);
query.set("collection", "testalias");
res = cluster.getSolrClient().query(query); // add one document to testalias, which will route to collection2 because it's the first
assertEquals(6, res.getResults().getNumFound()); new UpdateRequest()
.add("id", "12", "a_t", "humpty dumpy5 sat on a walls")
.commit(cluster.getSolrClient(), "testalias"); // thus gets added to collection2
searchSeveralWays("collection2", new SolrQuery("*:*"), 4);
CollectionAdminRequest.deleteAlias("testalias").process(cluster.getSolrClient()); CollectionAdminRequest.deleteAlias("testalias").process(cluster.getSolrClient());
CollectionAdminRequest.deleteAlias("testalias2").process(cluster.getSolrClient()); CollectionAdminRequest.deleteAlias("testalias2").process(cluster.getSolrClient());
@ -160,6 +124,49 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
assertTrue("Unexpected exception message: " + e.getMessage(), e.getMessage().contains("Collection not found: testalias")); assertTrue("Unexpected exception message: " + e.getMessage(), e.getMessage().contains("Collection not found: testalias"));
} }
private void searchSeveralWays(String collectionList, SolrParams solrQuery, int expectedNumFound) throws IOException, SolrServerException {
searchSeveralWays(collectionList, solrQuery, res -> assertEquals(expectedNumFound, res.getResults().getNumFound()));
}
private void searchSeveralWays(String collectionList, SolrParams solrQuery, Consumer<QueryResponse> responseConsumer) throws IOException, SolrServerException {
if (random().nextBoolean()) {
// cluster's CloudSolrClient
responseConsumer.accept(cluster.getSolrClient().query(collectionList, solrQuery));
} else {
// new CloudSolrClient (random shardLeadersOnly)
try (CloudSolrClient solrClient = getCloudSolrClient(cluster)) {
if (random().nextBoolean()) {
solrClient.setDefaultCollection(collectionList);
responseConsumer.accept(solrClient.query(null, solrQuery));
} else {
responseConsumer.accept(solrClient.query(collectionList, solrQuery));
}
}
}
// HttpSolrClient
JettySolrRunner jetty = cluster.getRandomJetty(random());
if (random().nextBoolean()) {
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/" + collectionList)) {
responseConsumer.accept(client.query(null, solrQuery));
}
} else {
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString())) {
responseConsumer.accept(client.query(collectionList, solrQuery));
}
}
// Recursively do again; this time with the &collection= param
if (solrQuery.get("collection") == null) {
// put in "collection" param
ModifiableSolrParams newParams = new ModifiableSolrParams(solrQuery);
newParams.set("collection", collectionList);
String maskedColl = new String[]{null, "bogus", "collection2", "collection1"}[random().nextInt(4)];
searchSeveralWays(maskedColl, newParams, responseConsumer);
}
}
public void testErrorChecks() throws Exception { public void testErrorChecks() throws Exception {
CollectionAdminRequest.createCollection("testErrorChecks-collection", "conf", 2, 1).process(cluster.getSolrClient()); CollectionAdminRequest.createCollection("testErrorChecks-collection", "conf", 2, 1).process(cluster.getSolrClient());
@ -189,6 +196,7 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
// Valid // Valid
CollectionAdminRequest.createAlias("testalias", "testErrorChecks-collection").process(cluster.getSolrClient()); CollectionAdminRequest.createAlias("testalias", "testErrorChecks-collection").process(cluster.getSolrClient());
// TODO dubious; remove?
CollectionAdminRequest.createAlias("testalias2", "testalias").process(cluster.getSolrClient()); CollectionAdminRequest.createAlias("testalias2", "testalias").process(cluster.getSolrClient());
// Alias + invalid // Alias + invalid

9
solr/core/src/test/org/apache/solr/cloud/DistribJoinFromCollectionTest.java
View File

@ -112,13 +112,13 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{
@Test @Test
public void testScore() throws Exception { public void testScore() throws Exception {
//without score //without score
testJoins(toColl, fromColl, toDocId, false); testJoins(toColl, fromColl, toDocId, true);
} }
@Test @Test
public void testNoScore() throws Exception { public void testNoScore() throws Exception {
//with score //with score
testJoins(toColl, fromColl, toDocId, true); testJoins(toColl, fromColl, toDocId, false);
} }
@ -141,7 +141,7 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{
} }
private void testJoins(String toColl, String fromColl, String toDocId, boolean isScoresTest) private void testJoins(String toColl, String fromColl, String toDocId, boolean isScoresTest)
throws SolrServerException, IOException { throws SolrServerException, IOException, InterruptedException {
// verify the join with fromIndex works // verify the join with fromIndex works
final String fromQ = "match_s:c^2"; final String fromQ = "match_s:c^2";
CloudSolrClient client = cluster.getSolrClient(); CloudSolrClient client = cluster.getSolrClient();
@ -164,8 +164,7 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{
// create an alias for the fromIndex and then query through the alias // create an alias for the fromIndex and then query through the alias
String alias = fromColl+"Alias"; String alias = fromColl+"Alias";
CollectionAdminRequest.CreateAlias request = CollectionAdminRequest.createAlias(alias,fromColl); CollectionAdminRequest.createAlias(alias, fromColl).process(client);
request.process(client);
{ {
final String joinQ = "{!join " + anyScoreMode(isScoresTest) final String joinQ = "{!join " + anyScoreMode(isScoresTest)

16
solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java
View File

@ -18,6 +18,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -53,20 +54,11 @@ public class DelegatingClusterStateProvider implements ClusterStateProvider {
} }
@Override @Override
public String getAlias(String alias) { public List<String> resolveAlias(String alias) {
if (delegate != null) { if (delegate != null) {
return delegate.getAlias(alias); return delegate.resolveAlias(alias);
} else { } else {
return null; return Collections.singletonList(alias);
}
}
@Override
public String getCollectionName(String name) {
if (delegate != null) {
return delegate.getCollectionName(name);
} else {
return null;
} }
} }

197
solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
View File

@ -27,6 +27,7 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Random; import java.util.Random;
@ -84,8 +85,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.slf4j.MDC; import org.slf4j.MDC;
import static org.apache.solr.common.params.CommonParams.ID;
import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS; import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS;
import static org.apache.solr.common.params.CommonParams.ID;
/** /**
* SolrJ client class to communicate with SolrCloud. * SolrJ client class to communicate with SolrCloud.
@ -204,6 +205,7 @@ public class CloudSolrClient extends SolrClient {
} }
protected final StateCache collectionStateCache = new StateCache(); protected final StateCache collectionStateCache = new StateCache();
class ExpiringCachedDocCollection { class ExpiringCachedDocCollection {
final DocCollection cached; final DocCollection cached;
final long cachedAt; final long cachedAt;
@ -222,7 +224,7 @@ public class CloudSolrClient extends SolrClient {
> TimeUnit.NANOSECONDS.convert(timeToLiveMs, TimeUnit.MILLISECONDS); > TimeUnit.NANOSECONDS.convert(timeToLiveMs, TimeUnit.MILLISECONDS);
} }
boolean shoulRetry() { boolean shouldRetry() {
if (maybeStale) {// we are not sure if it is stale so check with retry time if (maybeStale) {// we are not sure if it is stale so check with retry time
if ((retriedAt == -1 || if ((retriedAt == -1 ||
(System.nanoTime() - retriedAt) > retryExpiryTime)) { (System.nanoTime() - retriedAt) > retryExpiryTime)) {
@ -292,7 +294,7 @@ public class CloudSolrClient extends SolrClient {
} }
} }
/**Sets the cache ttl for DocCollection Objects cached . This is only applicable for collections which are persisted outside of clusterstate.json /** Sets the cache ttl for DocCollection Objects cached . This is only applicable for collections which are persisted outside of clusterstate.json
* @param seconds ttl value in seconds * @param seconds ttl value in seconds
*/ */
public void setCollectionCacheTTl(int seconds){ public void setCollectionCacheTTl(int seconds){
@ -455,7 +457,7 @@ public class CloudSolrClient extends SolrClient {
private NamedList<Object> directUpdate(AbstractUpdateRequest request, String collection) throws SolrServerException { private NamedList<Object> directUpdate(AbstractUpdateRequest request, String collection) throws SolrServerException {
UpdateRequest updateRequest = (UpdateRequest) request; UpdateRequest updateRequest = (UpdateRequest) request;
ModifiableSolrParams params = (ModifiableSolrParams) request.getParams(); SolrParams params = request.getParams();
ModifiableSolrParams routableParams = new ModifiableSolrParams(); ModifiableSolrParams routableParams = new ModifiableSolrParams();
ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams(); ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
@ -471,9 +473,9 @@ public class CloudSolrClient extends SolrClient {
throw new SolrServerException("No collection param specified on request and no default collection has been set."); throw new SolrServerException("No collection param specified on request and no default collection has been set.");
} }
//Check to see if the collection is an alias. //Check to see if the collection is an alias.
collection = stateProvider.getCollectionName(collection); List<String> aliasedCollections = stateProvider.resolveAlias(collection);
collection = aliasedCollections.get(0); // pick 1st (consistent with HttpSolrCall behavior)
DocCollection col = getDocCollection(collection, null); DocCollection col = getDocCollection(collection, null);
@ -786,11 +788,16 @@ public class CloudSolrClient extends SolrClient {
@Override @Override
public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException { public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException {
if (collection == null) { // the collection parameter of the request overrides that of the parameter to this method
collection = request.getCollection(); String requestCollection = request.getCollection();
if (collection == null) collection = defaultCollection; if (requestCollection != null) {
collection = requestCollection;
} else if (collection == null) {
collection = defaultCollection;
} }
return requestWithRetryOnStaleState(request, 0, collection); List<String> inputCollections =
collection == null ? Collections.emptyList() : StrUtils.splitSmart(collection, ",", true);
return requestWithRetryOnStaleState(request, 0, inputCollections);
} }
/** /**
@ -798,10 +805,8 @@ public class CloudSolrClient extends SolrClient {
* there's a chance that the request will fail due to cached stale state, * there's a chance that the request will fail due to cached stale state,
* which means the state must be refreshed from ZK and retried. * which means the state must be refreshed from ZK and retried.
*/ */
protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, String collection) protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, List<String> inputCollections)
throws SolrServerException, IOException { throws SolrServerException, IOException {
SolrRequest originalRequest = request;
connect(); // important to call this before you start working with the ZkStateReader connect(); // important to call this before you start working with the ZkStateReader
// build up a _stateVer_ param to pass to the server containing all of the // build up a _stateVer_ param to pass to the server containing all of the
@ -818,8 +823,8 @@ public class CloudSolrClient extends SolrClient {
isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest(); isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest();
} }
boolean isAdmin = ADMIN_PATHS.contains(request.getPath()); boolean isAdmin = ADMIN_PATHS.contains(request.getPath());
if (collection != null && !isAdmin && !isCollectionRequestOfV2) { // don't do _stateVer_ checking for admin, v2 api requests if (!inputCollections.isEmpty() && !isAdmin && !isCollectionRequestOfV2) { // don't do _stateVer_ checking for admin, v2 api requests
Set<String> requestedCollectionNames = getCollectionNames(collection); Set<String> requestedCollectionNames = resolveAliases(inputCollections);
StringBuilder stateVerParamBuilder = null; StringBuilder stateVerParamBuilder = null;
for (String requestedCollection : requestedCollectionNames) { for (String requestedCollection : requestedCollectionNames) {
@ -859,7 +864,7 @@ public class CloudSolrClient extends SolrClient {
NamedList<Object> resp = null; NamedList<Object> resp = null;
try { try {
resp = sendRequest(request, collection); resp = sendRequest(request, inputCollections);
//to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there //to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there
Object o = resp == null || resp.size() == 0 ? null : resp.get(STATE_VERSION, resp.size() - 1); Object o = resp == null || resp.size() == 0 ? null : resp.get(STATE_VERSION, resp.size() - 1);
if(o != null && o instanceof Map) { if(o != null && o instanceof Map) {
@ -878,7 +883,7 @@ public class CloudSolrClient extends SolrClient {
// don't do retry support for admin requests // don't do retry support for admin requests
// or if the request doesn't have a collection specified // or if the request doesn't have a collection specified
// or request is v2 api and its method is not GET // or request is v2 api and its method is not GET
if (collection == null || isAdmin || (request instanceof V2Request && request.getMethod() != SolrRequest.METHOD.GET)) { if (inputCollections.isEmpty() || isAdmin || (request instanceof V2Request && request.getMethod() != SolrRequest.METHOD.GET)) {
if (exc instanceof SolrServerException) { if (exc instanceof SolrServerException) {
throw (SolrServerException)exc; throw (SolrServerException)exc;
} else if (exc instanceof IOException) { } else if (exc instanceof IOException) {
@ -894,8 +899,8 @@ public class CloudSolrClient extends SolrClient {
int errorCode = (rootCause instanceof SolrException) ? int errorCode = (rootCause instanceof SolrException) ?
((SolrException)rootCause).code() : SolrException.ErrorCode.UNKNOWN.code; ((SolrException)rootCause).code() : SolrException.ErrorCode.UNKNOWN.code;
log.error("Request to collection {} failed due to ("+errorCode+ log.error("Request to collection {} failed due to (" + errorCode + ") {}, retry? " + retryCount,
") {}, retry? "+retryCount, collection, rootCause.toString()); inputCollections, rootCause.toString());
boolean wasCommError = boolean wasCommError =
(rootCause instanceof ConnectException || (rootCause instanceof ConnectException ||
@ -907,7 +912,7 @@ public class CloudSolrClient extends SolrClient {
// it was a communication error. it is likely that // it was a communication error. it is likely that
// the node to which the request to be sent is down . So , expire the state // the node to which the request to be sent is down . So , expire the state
// so that the next attempt would fetch the fresh state // so that the next attempt would fetch the fresh state
// just re-read state for all of them, if it has not been retired // just re-read state for all of them, if it has not been retried
// in retryExpiryTime time // in retryExpiryTime time
for (DocCollection ext : requestedCollections) { for (DocCollection ext : requestedCollections) {
ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(ext.getName()); ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(ext.getName());
@ -919,7 +924,7 @@ public class CloudSolrClient extends SolrClient {
// and we could not get any information from the server // and we could not get any information from the server
//it is probably not worth trying again and again because //it is probably not worth trying again and again because
// the state would not have been updated // the state would not have been updated
return requestWithRetryOnStaleState(request, retryCount + 1, collection); return requestWithRetryOnStaleState(request, retryCount + 1, inputCollections);
} }
} }
@ -963,8 +968,8 @@ public class CloudSolrClient extends SolrClient {
// if the state was stale, then we retry the request once with new state pulled from Zk // if the state was stale, then we retry the request once with new state pulled from Zk
if (stateWasStale) { if (stateWasStale) {
log.warn("Re-trying request to collection(s) "+collection+" after stale state error from server."); log.warn("Re-trying request to collection(s) "+inputCollections+" after stale state error from server.");
resp = requestWithRetryOnStaleState(request, retryCount+1, collection); resp = requestWithRetryOnStaleState(request, retryCount+1, inputCollections);
} else { } else {
if(exc instanceof SolrException) { if(exc instanceof SolrException) {
throw exc; throw exc;
@ -981,137 +986,97 @@ public class CloudSolrClient extends SolrClient {
return resp; return resp;
} }
protected NamedList<Object> sendRequest(SolrRequest request, String collection) protected NamedList<Object> sendRequest(SolrRequest request, List<String> inputCollections)
throws SolrServerException, IOException { throws SolrServerException, IOException {
connect(); connect();
boolean sendToLeaders = false; boolean sendToLeaders = false;
List<String> replicas = null;
if (request instanceof IsUpdateRequest) { if (request instanceof IsUpdateRequest) {
if (request instanceof UpdateRequest) { if (request instanceof UpdateRequest) {
String collection = inputCollections.isEmpty() ? null : inputCollections.get(0); // getting first mimics HttpSolrCall
NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection); NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection);
if (response != null) { if (response != null) {
return response; return response;
} }
} }
sendToLeaders = true; sendToLeaders = true;
replicas = new ArrayList<>();
} }
SolrParams reqParams = request.getParams(); SolrParams reqParams = request.getParams();
if (reqParams == null) { if (reqParams == null) { // TODO fix getParams to never return null!
reqParams = new ModifiableSolrParams(); reqParams = new ModifiableSolrParams();
} }
List<String> theUrlList = new ArrayList<>();
final Set<String> liveNodes = stateProvider.getLiveNodes();
final List<String> theUrlList = new ArrayList<>(); // we populate this as follows...
if (request instanceof V2Request) { if (request instanceof V2Request) {
Set<String> liveNodes = stateProvider.getLiveNodes();
if (!liveNodes.isEmpty()) { if (!liveNodes.isEmpty()) {
List<String> liveNodesList = new ArrayList<>(liveNodes); List<String> liveNodesList = new ArrayList<>(liveNodes);
Collections.shuffle(liveNodesList, rand); Collections.shuffle(liveNodesList, rand);
theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0), theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0),
(String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http"))); (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
} }
} else if (ADMIN_PATHS.contains(request.getPath())) { } else if (ADMIN_PATHS.contains(request.getPath())) {
Set<String> liveNodes = stateProvider.getLiveNodes();
for (String liveNode : liveNodes) { for (String liveNode : liveNodes) {
theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode, theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode,
(String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http"))); (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
} }
} else {
if (collection == null) {
throw new SolrServerException(
"No collection param specified on request and no default collection has been set.");
}
Set<String> collectionNames = getCollectionNames(collection); } else { // Typical...
if (collectionNames.size() == 0) { Set<String> collectionNames = resolveAliases(inputCollections);
if (collectionNames.isEmpty()) {
throw new SolrException(ErrorCode.BAD_REQUEST, throw new SolrException(ErrorCode.BAD_REQUEST,
"Could not find collection: " + collection); "No collection param specified on request and no default collection has been set: " + inputCollections);
} }
String shardKeys = reqParams.get(ShardParams._ROUTE_);
// TODO: not a big deal because of the caching, but we could avoid looking // TODO: not a big deal because of the caching, but we could avoid looking
// at every shard // at every shard when getting leaders if we tweaked some things
// when getting leaders if we tweaked some things
// Retrieve slices from the cloud state and, for each collection // Retrieve slices from the cloud state and, for each collection specified, add it to the Map of slices.
// specified,
// add it to the Map of slices.
Map<String,Slice> slices = new HashMap<>(); Map<String,Slice> slices = new HashMap<>();
String shardKeys = reqParams.get(ShardParams._ROUTE_);
for (String collectionName : collectionNames) { for (String collectionName : collectionNames) {
DocCollection col = getDocCollection(collectionName, null); DocCollection col = getDocCollection(collectionName, null);
Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams , col); Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams , col);
ClientUtils.addSlices(slices, collectionName, routeSlices, true); ClientUtils.addSlices(slices, collectionName, routeSlices, true);
} }
Set<String> liveNodes = stateProvider.getLiveNodes();
List<String> leaderUrlList = null; // Gather URLs, grouped by leader or replica
List<String> urlList = null;
List<String> replicasList = null;
// build a map of unique nodes
// TODO: allow filtering by group, role, etc // TODO: allow filtering by group, role, etc
Map<String,ZkNodeProps> nodes = new HashMap<>(); Set<String> seenNodes = new HashSet<>();
List<String> urlList2 = new ArrayList<>(); List<String> replicas = new ArrayList<>();
String joinedInputCollections = StrUtils.join(inputCollections, ',');
for (Slice slice : slices.values()) { for (Slice slice : slices.values()) {
for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) { for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps); ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
String node = coreNodeProps.getNodeName(); String node = coreNodeProps.getNodeName();
if (!liveNodes.contains(coreNodeProps.getNodeName()) if (!liveNodes.contains(node) // Must be a live node to continue
|| Replica.State.getState(coreNodeProps.getState()) != Replica.State.ACTIVE) continue; || Replica.State.getState(coreNodeProps.getState()) != Replica.State.ACTIVE) // Must be an ACTIVE replica to continue
if (nodes.put(node, nodeProps) == null) { continue;
if (!sendToLeaders || coreNodeProps.isLeader()) { if (seenNodes.add(node)) { // if we haven't yet collected a URL to this node...
String url; String url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), joinedInputCollections);
if (reqParams.get(UpdateParams.COLLECTION) == null) { if (sendToLeaders && coreNodeProps.isLeader()) {
url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection); theUrlList.add(url); // put leaders here eagerly (if sendToLeader mode)
} else {
url = coreNodeProps.getCoreUrl();
}
urlList2.add(url);
} else { } else {
String url; replicas.add(url); // replicas here
if (reqParams.get(UpdateParams.COLLECTION) == null) {
url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
} else {
url = coreNodeProps.getCoreUrl();
}
replicas.add(url);
} }
} }
} }
} }
if (sendToLeaders) {
leaderUrlList = urlList2;
replicasList = replicas;
} else {
urlList = urlList2;
}
if (sendToLeaders) {
theUrlList = new ArrayList<>(leaderUrlList.size());
theUrlList.addAll(leaderUrlList);
} else {
theUrlList = new ArrayList<>(urlList.size());
theUrlList.addAll(urlList);
}
// Shuffle the leaders, if any (none if !sendToLeaders)
Collections.shuffle(theUrlList, rand); Collections.shuffle(theUrlList, rand);
if (sendToLeaders) {
ArrayList<String> theReplicas = new ArrayList<>( // Shuffle the replicas, if any, and append to our list
replicasList.size()); Collections.shuffle(replicas, rand);
theReplicas.addAll(replicasList); theUrlList.addAll(replicas);
Collections.shuffle(theReplicas, rand);
theUrlList.addAll(theReplicas);
}
if (theUrlList.isEmpty()) { if (theUrlList.isEmpty()) {
for (String s : collectionNames) { collectionStateCache.keySet().removeAll(collectionNames);
if (s != null) collectionStateCache.remove(s);
}
throw new SolrException(SolrException.ErrorCode.INVALID_STATE, throw new SolrException(SolrException.ErrorCode.INVALID_STATE,
"Could not find a healthy node to handle the request."); "Could not find a healthy node to handle the request.");
} }
@ -1122,24 +1087,20 @@ public class CloudSolrClient extends SolrClient {
return rsp.getResponse(); return rsp.getResponse();
} }
private Set<String> getCollectionNames(String collection) { /** Resolves the input collections to their possible aliased collections. Doesn't validate collection existence. */
// Extract each comma separated collection name and store in a List. private LinkedHashSet<String> resolveAliases(List<String> inputCollections) {
List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true); LinkedHashSet<String> collectionNames = new LinkedHashSet<>(); // consistent ordering
Set<String> collectionNames = new HashSet<>(); for (String collectionName : inputCollections) {
// validate collections
for (String collectionName : rawCollectionsList) {
if (stateProvider.getState(collectionName) == null) { if (stateProvider.getState(collectionName) == null) {
String alias = stateProvider.getAlias(collectionName); // perhaps it's an alias
if (alias != null) { List<String> aliasedCollections = stateProvider.resolveAlias(collectionName);
List<String> aliasList = StrUtils.splitSmart(alias, ",", true); // one more level of alias indirection... (dubious that we should support this)
collectionNames.addAll(aliasList); for (String aliasedCollection : aliasedCollections) {
continue; collectionNames.addAll(stateProvider.resolveAlias(aliasedCollection));
} }
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName); collectionNames.add(collectionName); // it's a collection
} }
collectionNames.add(collectionName);
} }
return collectionNames; return collectionNames;
} }
@ -1199,7 +1160,7 @@ public class CloudSolrClient extends SolrClient {
DocCollection col = cacheEntry == null ? null : cacheEntry.cached; DocCollection col = cacheEntry == null ? null : cacheEntry.cached;
if (col != null) { if (col != null) {
if (expectedVersion <= col.getZNodeVersion() if (expectedVersion <= col.getZNodeVersion()
&& !cacheEntry.shoulRetry()) return col; && !cacheEntry.shouldRetry()) return col;
} }
CollectionRef ref = getCollectionRef(collection); CollectionRef ref = getCollectionRef(collection);
@ -1220,7 +1181,7 @@ public class CloudSolrClient extends SolrClient {
col = cacheEntry == null ? null : cacheEntry.cached; col = cacheEntry == null ? null : cacheEntry.cached;
if (col != null) { if (col != null) {
if (expectedVersion <= col.getZNodeVersion() if (expectedVersion <= col.getZNodeVersion()
&& !cacheEntry.shoulRetry()) return col; && !cacheEntry.shouldRetry()) return col;
} }
// We are going to fetch a new version // We are going to fetch a new version
// we MUST try to get a new version // we MUST try to get a new version
@ -1466,7 +1427,7 @@ public class CloudSolrClient extends SolrClient {
} }
/** /**
* Tells {@link Builder} that created clients should send updats only to shard leaders. * Tells {@link Builder} that created clients should send updates only to shard leaders.
*/ */
public Builder sendUpdatesOnlyToShardLeaders() { public Builder sendUpdatesOnlyToShardLeaders() {
shardLeadersOnly = true; shardLeadersOnly = true;

12
solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.impl;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.util.Map; import java.util.Map;
import java.util.List;
import java.util.Set; import java.util.Set;
import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterState;
@ -37,15 +38,10 @@ public interface ClusterStateProvider extends Closeable {
Set<String> getLiveNodes(); Set<String> getLiveNodes();
/** /**
* Given an alias, returns the collection name that this alias points to * Given a collection alias, returns a list of collections it points to, or returns a singleton list of the input if
* it's not an alias.
*/ */
String getAlias(String alias); List<String> resolveAlias(String alias);
/**
* Given a name, returns the collection name if an alias by that name exists, or
* returns the name itself, if no alias exists.
*/
String getCollectionName(String name);
/** /**
* Obtain the current cluster state. * Obtain the current cluster state.

20
solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java
View File

@ -32,6 +32,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ClusterState.CollectionRef; import org.apache.solr.common.cloud.ClusterState.CollectionRef;
import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZkStateReader;
@ -47,7 +48,7 @@ public class HttpClusterStateProvider implements ClusterStateProvider {
private String urlScheme; private String urlScheme;
volatile Set<String> liveNodes; volatile Set<String> liveNodes;
long liveNodesTimestamp = 0; long liveNodesTimestamp = 0;
volatile Map<String, String> aliases; volatile Map<String, List<String>> aliases;
long aliasesTimestamp = 0; long aliasesTimestamp = 0;
private int cacheTimeout = 5; // the liveNodes and aliases cache will be invalidated after 5 secs private int cacheTimeout = 5; // the liveNodes and aliases cache will be invalidated after 5 secs
@ -191,12 +192,11 @@ public class HttpClusterStateProvider implements ClusterStateProvider {
} }
@Override @Override
public String getAlias(String alias) { public List<String> resolveAlias(String aliasName) {
Map<String, String> aliases = getAliases(false); return Aliases.resolveAliasesGivenAliasMap(getAliases(false), aliasName);
return aliases.get(alias);
} }
private Map<String, String> getAliases(boolean forceFetch) { private Map<String, List<String>> getAliases(boolean forceFetch) {
if (this.liveNodes == null) { if (this.liveNodes == null) {
throw new RuntimeException("We don't know of any live_nodes to fetch the" throw new RuntimeException("We don't know of any live_nodes to fetch the"
+ " latest aliases information from. " + " latest aliases information from. "
@ -212,10 +212,10 @@ public class HttpClusterStateProvider implements ClusterStateProvider {
withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)). withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)).
withHttpClient(httpClient).build()) { withHttpClient(httpClient).build()) {
Map<String, String> aliases = new CollectionAdminRequest.ListAliases().process(client).getAliases(); Map<String, List<String>> aliases = new CollectionAdminRequest.ListAliases().process(client).getAliasesAsLists();
this.aliases = aliases; this.aliases = aliases;
this.aliasesTimestamp = System.nanoTime(); this.aliasesTimestamp = System.nanoTime();
return Collections.unmodifiableMap(aliases); return Collections.unmodifiableMap(this.aliases);
} catch (SolrServerException | RemoteSolrException | IOException e) { } catch (SolrServerException | RemoteSolrException | IOException e) {
// Situation where we're hitting an older Solr which doesn't have LISTALIASES // Situation where we're hitting an older Solr which doesn't have LISTALIASES
if (e instanceof RemoteSolrException && ((RemoteSolrException)e).code()==400) { if (e instanceof RemoteSolrException && ((RemoteSolrException)e).code()==400) {
@ -240,12 +240,6 @@ public class HttpClusterStateProvider implements ClusterStateProvider {
} }
} }
@Override
public String getCollectionName(String name) {
Map<String, String> aliases = getAliases(false);
return aliases.containsKey(name) ? aliases.get(name): name;
}
@Override @Override
public ClusterState getClusterState() throws IOException { public ClusterState getClusterState() throws IOException {
for (String nodeName: liveNodes) { for (String nodeName: liveNodes) {

19
solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java
View File

@ -22,11 +22,11 @@ import java.lang.invoke.MethodHandles;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.solr.common.cloud.ZooKeeperException;
@ -83,9 +83,8 @@ public class ZkClientClusterStateProvider implements ClusterStateProvider {
@Override @Override
public String getAlias(String alias) { public List<String> resolveAlias(String alias) {
Aliases aliases = zkStateReader.getAliases(); return zkStateReader.getAliases().resolveAliases(alias); // if not an alias, returns itself
return aliases.getCollectionAlias(alias);
} }
@Override @Override
@ -103,18 +102,6 @@ public class ZkClientClusterStateProvider implements ClusterStateProvider {
return def; return def;
} }
@Override
public String getCollectionName(String name) {
Aliases aliases = zkStateReader.getAliases();
if (aliases != null) {
Map<String, String> collectionAliases = aliases.getCollectionAliasMap();
if (collectionAliases != null && collectionAliases.containsKey(name)) {
name = collectionAliases.get(name);
}
}
return name;
}
@Override @Override
public ClusterState getClusterState() throws IOException { public ClusterState getClusterState() throws IOException {
return zkStateReader.getClusterState(); return zkStateReader.getClusterState();

38
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java
View File

@ -26,6 +26,7 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Objects;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
@ -46,7 +47,6 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.Slice;
@ -55,7 +55,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.common.util.StrUtils;
import static org.apache.solr.common.params.CommonParams.DISTRIB; import static org.apache.solr.common.params.CommonParams.DISTRIB;
import static org.apache.solr.common.params.CommonParams.SORT; import static org.apache.solr.common.params.CommonParams.SORT;
@ -331,9 +330,21 @@ public class CloudSolrStream extends TupleStream implements Expressible {
Map<String, DocCollection> collectionsMap = clusterState.getCollectionsMap(); Map<String, DocCollection> collectionsMap = clusterState.getCollectionsMap();
// Check collection case sensitive //TODO we should probably split collection by comma to query more than one
if(collectionsMap.containsKey(collectionName)) { // which is something already supported in other parts of Solr
return collectionsMap.get(collectionName).getActiveSlices();
// check for alias or collection
List<String> collections = checkAlias
? zkStateReader.getAliases().resolveAliases(collectionName) // if not an alias, returns collectionName
: Collections.singletonList(collectionName);
// Lookup all actives slices for these collections
List<Slice> slices = collections.stream()
.map(collectionsMap::get)
.filter(Objects::nonNull)
.flatMap(docCol -> docCol.getActiveSlices().stream())
.collect(Collectors.toList());
if (!slices.isEmpty()) {
return slices;
} }
// Check collection case insensitive // Check collection case insensitive
@ -343,23 +354,6 @@ public class CloudSolrStream extends TupleStream implements Expressible {
} }
} }
if(checkAlias) {
// check for collection alias
Aliases aliases = zkStateReader.getAliases();
String alias = aliases.getCollectionAlias(collectionName);
if (alias != null) {
Collection<Slice> slices = new ArrayList<>();
List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
for (String aliasCollectionName : aliasList) {
// Add all active slices for this alias collection
slices.addAll(collectionsMap.get(aliasCollectionName).getActiveSlices());
}
return slices;
}
}
throw new IOException("Slices not found for " + collectionName); throw new IOException("Slices not found for " + collectionName);
} }

44
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupleStream.java
View File

@ -37,14 +37,11 @@ import org.apache.solr.client.solrj.io.stream.expr.Explanation;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
import org.apache.solr.common.IteratorWriter; import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter; import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.StrUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -141,7 +138,7 @@ public abstract class TupleStream implements Closeable, Serializable, MapWriter
CloudSolrClient cloudSolrClient = streamContext.getSolrClientCache().getCloudSolrClient(zkHost); CloudSolrClient cloudSolrClient = streamContext.getSolrClientCache().getCloudSolrClient(zkHost);
ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState(); ClusterState clusterState = zkStateReader.getClusterState();
Collection<Slice> slices = getSlices(collection, zkStateReader, true); Collection<Slice> slices = CloudSolrStream.getSlices(collection, zkStateReader, true);
Set<String> liveNodes = clusterState.getLiveNodes(); Set<String> liveNodes = clusterState.getLiveNodes();
for(Slice slice : slices) { for(Slice slice : slices) {
Collection<Replica> replicas = slice.getReplicas(); Collection<Replica> replicas = slice.getReplicas();
@ -162,45 +159,6 @@ public abstract class TupleStream implements Closeable, Serializable, MapWriter
return shards; return shards;
} }
public static Collection<Slice> getSlices(String collectionName,
ZkStateReader zkStateReader,
boolean checkAlias) throws IOException {
ClusterState clusterState = zkStateReader.getClusterState();
Map<String, DocCollection> collectionsMap = clusterState.getCollectionsMap();
// Check collection case sensitive
if(collectionsMap.containsKey(collectionName)) {
return collectionsMap.get(collectionName).getActiveSlices();
}
// Check collection case insensitive
for(String collectionMapKey : collectionsMap.keySet()) {
if(collectionMapKey.equalsIgnoreCase(collectionName)) {
return collectionsMap.get(collectionMapKey).getActiveSlices();
}
}
if(checkAlias) {
// check for collection alias
Aliases aliases = zkStateReader.getAliases();
String alias = aliases.getCollectionAlias(collectionName);
if (alias != null) {
Collection<Slice> slices = new ArrayList<>();
List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
for (String aliasCollectionName : aliasList) {
// Add all active slices for this alias collection
slices.addAll(collectionsMap.get(aliasCollectionName).getActiveSlices());
}
return slices;
}
}
throw new IOException("Slices not found for " + collectionName);
}
public static class IgnoreException extends IOException { public static class IgnoreException extends IOException {
public void printStackTrace(PrintWriter pw) { public void printStackTrace(PrintWriter pw) {
pw.print("Early Client Disconnect"); pw.print("Early Client Disconnect");

7
solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java
View File

@ -18,8 +18,10 @@ package org.apache.solr.client.solrj.response;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
public class CollectionAdminResponse extends SolrResponseBase public class CollectionAdminResponse extends SolrResponseBase
@ -76,6 +78,11 @@ public class CollectionAdminResponse extends SolrResponseBase
return Collections.emptyMap(); return Collections.emptyMap();
} }
public Map<String, List<String>> getAliasesAsLists() {
// TODO we compute on each call... should this be done once & cached?
return Aliases.convertMapOfCommaDelimitedToMapOfList(getAliases());
}
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Map<String, NamedList<Integer>> getCollectionNodesStatus() public Map<String, NamedList<Integer>> getCollectionNodesStatus()
{ {

129
solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
View File

@ -16,47 +16,132 @@
*/ */
package org.apache.solr.common.cloud; package org.apache.solr.common.cloud;
import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
/**
* Holds collection aliases -- virtual collections that point to one or more other collections.
* We might add other types of aliases here some day.
* Immutable.
*/
public class Aliases { public class Aliases {
private Map<String,Map<String,String>> aliasMap; public static final Aliases EMPTY = new Aliases(Collections.emptyMap());
public Aliases(Map<String,Map<String,String>> aliasMap) { /** Map of "collection" string constant to ->
* alias name -> comma delimited list of collections */
private final Map<String,Map<String,String>> aliasMap; // not-null
private final Map<String, List<String>> collectionAliasListMap; // not-null; computed from aliasMap
public static Aliases fromJSON(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return EMPTY;
}
return new Aliases((Map<String,Map<String,String>>) Utils.fromJSON(bytes));
}
private Aliases(Map<String, Map<String,String>> aliasMap) {
this.aliasMap = aliasMap; this.aliasMap = aliasMap;
collectionAliasListMap = convertMapOfCommaDelimitedToMapOfList(getCollectionAliasMap());
} }
public Aliases() { public static Map<String, List<String>> convertMapOfCommaDelimitedToMapOfList(Map<String, String> collectionAliasMap) {
this.aliasMap = new HashMap<>(); Map<String, List<String>> collectionAliasListMap = new LinkedHashMap<>(collectionAliasMap.size());
for (Map.Entry<String, String> entry : collectionAliasMap.entrySet()) {
collectionAliasListMap.put(entry.getKey(), StrUtils.splitSmart(entry.getValue(), ",", true));
}
return collectionAliasListMap;
} }
/**
* Returns an unmodifiable Map of collection aliases mapped to a comma delimited list of what the alias maps to.
* Does not return null.
* Prefer use of {@link #getCollectionAliasListMap()} instead, where appropriate.
*/
public Map<String,String> getCollectionAliasMap() { public Map<String,String> getCollectionAliasMap() {
Map<String,String> cam = aliasMap.get("collection"); Map<String,String> cam = aliasMap.get("collection");
if (cam == null) return null; return cam == null ? Collections.emptyMap() : Collections.unmodifiableMap(cam);
return Collections.unmodifiableMap(cam);
}
public Map<String,Map<String,String>> getAliasMap() {
return Collections.unmodifiableMap(aliasMap);
} }
public int collectionAliasSize() { /**
Map<String,String> cam = aliasMap.get("collection"); * Returns an unmodifiable Map of collection aliases mapped to a list of what the alias maps to.
if (cam == null) return 0; * Does not return null.
return cam.size(); */
public Map<String,List<String>> getCollectionAliasListMap() {
return Collections.unmodifiableMap(collectionAliasListMap);
} }
public boolean hasCollectionAliases() {
return !collectionAliasListMap.isEmpty();
}
/**
* Returns a list of collections that the input alias name maps to. If there
* are none, the input is returned. One level of alias indirection is supported (alias to alias to collection).
* Treat the result as unmodifiable.
*/
public List<String> resolveAliases(String aliasName) {
return resolveAliasesGivenAliasMap(collectionAliasListMap, aliasName);
}
/** @lucene.internal */
public static List<String> resolveAliasesGivenAliasMap(Map<String, List<String>> collectionAliasListMap, String aliasName) {
//return collectionAliasListMap.getOrDefault(aliasName, Collections.singletonList(aliasName));
// TODO deprecate and remove this dubious feature?
// Due to another level of indirection, this is more complicated...
List<String> level1 = collectionAliasListMap.get(aliasName);
if (level1 == null) {
return Collections.singletonList(aliasName);// is a collection
}
List<String> result = new ArrayList<>(level1.size());
for (String level1Alias : level1) {
List<String> level2 = collectionAliasListMap.get(level1Alias);
if (level2 == null) {
result.add(level1Alias);
} else {
result.addAll(level2);
}
}
return result;
}
/**
* Creates a new Aliases instance with the same data as the current one but with a modification based on the
* parameters. If {@code collections} is null, then the {@code alias} is removed, otherwise it is added/updated.
*/
public Aliases cloneWithCollectionAlias(String alias, String collections) {
Map<String,String> newCollectionMap = new HashMap<>(getCollectionAliasMap());
if (collections == null) {
newCollectionMap.remove(alias);
} else {
newCollectionMap.put(alias, collections);
}
if (newCollectionMap.isEmpty()) {
return EMPTY;
} else {
return new Aliases(Collections.singletonMap("collection", newCollectionMap));
}
}
/** Serialize to ZooKeeper. */
public byte[] toJSON() {
if (collectionAliasListMap.isEmpty()) {
return null;
} else {
return Utils.toJSON(aliasMap);
}
}
@Override @Override
public String toString() { public String toString() {
return "Aliases [aliasMap=" + aliasMap + "]"; return "Aliases [aliasMap=" + aliasMap + "]";
} }
public String getCollectionAlias(String collectionName) {
Map<String,String> cam = aliasMap.get("collection");
if (cam == null) return null;
return cam.get(collectionName);
}
} }

9
solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
View File

@ -232,15 +232,6 @@ public class ClusterState implements JSONWriter.Writable {
return new ClusterState( liveNodes, collections,version); return new ClusterState( liveNodes, collections,version);
} }
public static Aliases load(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return new Aliases();
}
Map<String,Map<String,String>> aliasMap = (Map<String,Map<String,String>>) Utils.fromJSON(bytes);
return new Aliases(aliasMap);
}
// TODO move to static DocCollection.loadFromMap // TODO move to static DocCollection.loadFromMap
private static DocCollection collectionFromObjects(String name, Map<String, Object> objs, Integer version, String znode) { private static DocCollection collectionFromObjects(String name, Map<String, Object> objs, Integer version, String znode) {
Map<String,Object> props; Map<String,Object> props;

10
solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
View File

@ -257,7 +257,7 @@ public class ZkStateReader implements Closeable {
private final boolean closeClient; private final boolean closeClient;
private volatile Aliases aliases = new Aliases(); private volatile Aliases aliases = Aliases.EMPTY;
private volatile boolean closed = false; private volatile boolean closed = false;
@ -385,8 +385,10 @@ public class ZkStateReader implements Closeable {
public void updateLiveNodes() throws KeeperException, InterruptedException { public void updateLiveNodes() throws KeeperException, InterruptedException {
refreshLiveNodes(null); refreshLiveNodes(null);
} }
/** Never null. */
public Aliases getAliases() { public Aliases getAliases() {
assert aliases != null;
return aliases; return aliases;
} }
@ -455,7 +457,7 @@ public class ZkStateReader implements Closeable {
final Watcher thisWatch = this; final Watcher thisWatch = this;
final Stat stat = new Stat(); final Stat stat = new Stat();
final byte[] data = zkClient.getData(ALIASES, thisWatch, stat, true); final byte[] data = zkClient.getData(ALIASES, thisWatch, stat, true);
ZkStateReader.this.aliases = ClusterState.load(data); ZkStateReader.this.aliases = Aliases.fromJSON(data);
LOG.debug("New alias definition is: " + ZkStateReader.this.aliases.toString()); LOG.debug("New alias definition is: " + ZkStateReader.this.aliases.toString());
} }
} catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
@ -896,7 +898,7 @@ public class ZkStateReader implements Closeable {
public void updateAliases() throws KeeperException, InterruptedException { public void updateAliases() throws KeeperException, InterruptedException {
final byte[] data = zkClient.getData(ALIASES, null, null, true); final byte[] data = zkClient.getData(ALIASES, null, null, true);
this.aliases = ClusterState.load(data); this.aliases = Aliases.fromJSON(data);
} }
/** /**

1
solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
View File

@ -88,6 +88,7 @@ public class StrUtils {
* @param s the string to split * @param s the string to split
* @param separator the separator to split on * @param separator the separator to split on
* @param decode decode backslash escaping * @param decode decode backslash escaping
* @return not null
*/ */
public static List<String> splitSmart(String s, String separator, boolean decode) { public static List<String> splitSmart(String s, String separator, boolean decode) {
ArrayList<String> lst = new ArrayList<>(2); ArrayList<String> lst = new ArrayList<>(2);

10
solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
View File

@ -22,6 +22,7 @@ import java.net.SocketException;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@ -130,13 +131,8 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 {
} }
@Override @Override
public String getAlias(String collection) { public List<String> resolveAlias(String collection) {
return collection; return Collections.singletonList(collection);
}
@Override
public String getCollectionName(String name) {
return name;
} }
@Override @Override

9
solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
View File

@ -28,7 +28,6 @@ import java.sql.Types;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.Set; import java.util.Set;
import java.util.SortedSet; import java.util.SortedSet;
@ -555,13 +554,7 @@ public class JdbcTest extends SolrCloudTestCase {
tables.addAll(collectionsSet); tables.addAll(collectionsSet);
Aliases aliases = zkStateReader.getAliases(); Aliases aliases = zkStateReader.getAliases();
if(aliases != null) { tables.addAll(aliases.getCollectionAliasListMap().keySet());
Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap();
if(collectionAliasMap != null) {
Set<String> aliasesSet = collectionAliasMap.keySet();
tables.addAll(aliasesSet);
}
}
try(ResultSet rs = databaseMetaData.getTables(null, zkHost, "%", null)) { try(ResultSet rs = databaseMetaData.getTables(null, zkHost, "%", null)) {
for(String table : tables) { for(String table : tables) {

32
solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
View File

@ -84,6 +84,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient; import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.IpTables; import org.apache.solr.cloud.IpTables;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
@ -2224,6 +2225,14 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
super(); super();
} }
public CloudSolrClient.Builder withCluster(MiniSolrCloudCluster cluster) {
if (random().nextBoolean()) {
return withZkHost(cluster.getZkServer().getZkAddress());
} else {
return withSolrUrl(cluster.getRandomJetty(random()).getBaseUrl().toString());
}
}
@Override @Override
public CloudSolrClient.Builder sendDirectUpdatesToShardLeadersOnly() { public CloudSolrClient.Builder sendDirectUpdatesToShardLeadersOnly() {
configuredDUTflag = true; configuredDUTflag = true;
@ -2247,7 +2256,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
@Override @Override
public CloudSolrClient build() { public CloudSolrClient build() {
if (configuredDUTflag == false) { if (configuredDUTflag == false) {
// flag value not explicity configured // flag value not explicitly configured
if (random().nextBoolean()) { if (random().nextBoolean()) {
// so randomly choose a value // so randomly choose a value
randomlyChooseDirectUpdatesToLeadersOnly(); randomlyChooseDirectUpdatesToLeadersOnly();
@ -2262,7 +2271,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
/** /**
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient. * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
* Tests that do not wish to have any randomized behavior should use the * Tests that do not wish to have any randomized behavior should use the
* {@link org.apache.solr.client.solrj.impl.CloudSolrClient.Builder} class directly * {@link org.apache.solr.client.solrj.impl.CloudSolrClient.Builder} class directly
*/ */
public static CloudSolrClient getCloudSolrClient(String zkHost) { public static CloudSolrClient getCloudSolrClient(String zkHost) {
@ -2270,7 +2279,18 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
.withZkHost(zkHost) .withZkHost(zkHost)
.build(); .build();
} }
/**
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
* Tests that do not wish to have any randomized behavior should use the
* {@link org.apache.solr.client.solrj.impl.CloudSolrClient.Builder} class directly
*/
public static CloudSolrClient getCloudSolrClient(MiniSolrCloudCluster cluster) {
return new CloudSolrClientBuilder()
.withCluster(cluster)
.build();
}
/** /**
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient. * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
* Tests that do not wish to have any randomized behavior should use the * Tests that do not wish to have any randomized behavior should use the
@ -2300,7 +2320,11 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
.sendUpdatesToAllReplicasInShard() .sendUpdatesToAllReplicasInShard()
.build(); .build();
} }
public static CloudSolrClientBuilder newCloudSolrClient(String zkHost) {
return (CloudSolrClientBuilder) new CloudSolrClientBuilder().withZkHost(zkHost);
}
/** /**
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient. * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
* Tests that do not wish to have any randomized behavior should use the * Tests that do not wish to have any randomized behavior should use the