mirror of https://github.com/apache/lucene.git
SOLR-11444: Improve consistency of collection alias handling and collection list references.
Other refactorings of nearby code too.
This commit is contained in:
parent
99e853faf8
commit
e001f35289
|
@ -79,6 +79,11 @@ Other Changes
|
||||||
|
|
||||||
* SOLR-11464, SOLR-11493: Minor refactorings to DistributedUpdateProcessor. (Gus Heck, David Smiley)
|
* SOLR-11464, SOLR-11493: Minor refactorings to DistributedUpdateProcessor. (Gus Heck, David Smiley)
|
||||||
|
|
||||||
|
* SOLR-11444: Improved consistency of collection alias handling, and collection references that are
|
||||||
|
comma delimited lists of collections, across the various places collections can be referred to.
|
||||||
|
Updates are delivered to the first in a list. It's now possible to refer to a comma delimited list
|
||||||
|
of them in the path, ex: /solr/colA,colB/select?... (David Smiley)
|
||||||
|
|
||||||
================== 7.1.0 ==================
|
================== 7.1.0 ==================
|
||||||
|
|
||||||
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
|
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
|
||||||
|
|
|
@ -53,6 +53,7 @@ import org.apache.solr.common.util.PathTrie;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
|
||||||
import static org.apache.solr.common.params.CommonParams.JSON;
|
import static org.apache.solr.common.params.CommonParams.JSON;
|
||||||
import static org.apache.solr.common.params.CommonParams.WT;
|
import static org.apache.solr.common.params.CommonParams.WT;
|
||||||
import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN;
|
import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN;
|
||||||
|
@ -105,28 +106,30 @@ public class V2HttpCall extends HttpSolrCall {
|
||||||
}
|
}
|
||||||
|
|
||||||
if ("c".equals(prefix) || "collections".equals(prefix)) {
|
if ("c".equals(prefix) || "collections".equals(prefix)) {
|
||||||
String collectionName = origCorename = corename = pieces.get(1);
|
origCorename = pieces.get(1);
|
||||||
|
|
||||||
|
collectionsList = resolveCollectionListOrAlias(queryParams.get(COLLECTION_PROP, origCorename));
|
||||||
|
String collectionName = collectionsList.get(0); // first
|
||||||
|
//TODO try the other collections if can't find a local replica of the first?
|
||||||
|
|
||||||
DocCollection collection = getDocCollection(collectionName);
|
DocCollection collection = getDocCollection(collectionName);
|
||||||
if (collection == null) {
|
if (collection == null) {
|
||||||
if ( ! path.endsWith(CommonParams.INTROSPECT)) {
|
if ( ! path.endsWith(CommonParams.INTROSPECT)) {
|
||||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such collection or alias");
|
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such collection or alias");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
boolean isPreferLeader = false;
|
boolean isPreferLeader = (path.endsWith("/update") || path.contains("/update/"));
|
||||||
if (path.endsWith("/update") || path.contains("/update/")) {
|
|
||||||
isPreferLeader = true;
|
|
||||||
}
|
|
||||||
core = getCoreByCollection(collection.getName(), isPreferLeader);
|
core = getCoreByCollection(collection.getName(), isPreferLeader);
|
||||||
if (core == null) {
|
if (core == null) {
|
||||||
//this collection exists , but this node does not have a replica for that collection
|
//this collection exists , but this node does not have a replica for that collection
|
||||||
//todo find a better way to compute remote
|
//todo find a better way to compute remote
|
||||||
extractRemotePath(corename, origCorename, 0);
|
extractRemotePath(collectionName, origCorename, 0);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if ("cores".equals(prefix)) {
|
} else if ("cores".equals(prefix)) {
|
||||||
origCorename = corename = pieces.get(1);
|
origCorename = pieces.get(1);
|
||||||
core = cores.getCore(corename);
|
core = cores.getCore(origCorename);
|
||||||
}
|
}
|
||||||
if (core == null) {
|
if (core == null) {
|
||||||
log.error(">> path: '" + path + "'");
|
log.error(">> path: '" + path + "'");
|
||||||
|
@ -134,7 +137,7 @@ public class V2HttpCall extends HttpSolrCall {
|
||||||
initAdminRequest(path);
|
initAdminRequest(path);
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no core retrieved for " + corename);
|
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no core retrieved for " + origCorename);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -148,9 +151,7 @@ public class V2HttpCall extends HttpSolrCall {
|
||||||
MDCLoggingContext.setCore(core);
|
MDCLoggingContext.setCore(core);
|
||||||
parseRequest();
|
parseRequest();
|
||||||
|
|
||||||
if (usingAliases) {
|
addCollectionParamIfNeeded(getCollectionsList());
|
||||||
processAliases(aliases, collectionsList);
|
|
||||||
}
|
|
||||||
|
|
||||||
action = PROCESS;
|
action = PROCESS;
|
||||||
// we are done with a valid handler
|
// we are done with a valid handler
|
||||||
|
@ -180,17 +181,12 @@ public class V2HttpCall extends HttpSolrCall {
|
||||||
if (solrReq == null) solrReq = parser.parse(core, path, req);
|
if (solrReq == null) solrReq = parser.parse(core, path, req);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected DocCollection getDocCollection(String collectionName) {
|
protected DocCollection getDocCollection(String collectionName) { // note: don't send an alias; resolve it first
|
||||||
if (!cores.isZooKeeperAware()) {
|
if (!cores.isZooKeeperAware()) {
|
||||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Solr not running in cloud mode ");
|
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Solr not running in cloud mode ");
|
||||||
}
|
}
|
||||||
ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
|
ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
|
||||||
DocCollection collection = zkStateReader.getClusterState().getCollectionOrNull(collectionName);
|
return zkStateReader.getClusterState().getCollectionOrNull(collectionName);
|
||||||
if (collection == null) {
|
|
||||||
collectionName = corename = lookupAliases(collectionName);
|
|
||||||
collection = zkStateReader.getClusterState().getCollectionOrNull(collectionName);
|
|
||||||
}
|
|
||||||
return collection;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Api getApiInfo(PluginBag<SolrRequestHandler> requestHandlers,
|
public static Api getApiInfo(PluginBag<SolrRequestHandler> requestHandlers,
|
||||||
|
|
|
@ -17,27 +17,28 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.solr.cloud;
|
package org.apache.solr.cloud;
|
||||||
|
|
||||||
import static org.apache.solr.common.params.CommonParams.NAME;
|
|
||||||
|
|
||||||
import java.lang.invoke.MethodHandles;
|
import java.lang.invoke.MethodHandles;
|
||||||
import java.util.HashMap;
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
|
import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
|
||||||
import org.apache.solr.common.SolrException;
|
import org.apache.solr.common.SolrException;
|
||||||
import org.apache.solr.common.cloud.Aliases;
|
|
||||||
import org.apache.solr.common.cloud.ClusterState;
|
import org.apache.solr.common.cloud.ClusterState;
|
||||||
import org.apache.solr.common.cloud.ZkNodeProps;
|
import org.apache.solr.common.cloud.ZkNodeProps;
|
||||||
import org.apache.solr.common.cloud.ZkStateReader;
|
import org.apache.solr.common.cloud.ZkStateReader;
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
import org.apache.solr.common.util.Utils;
|
import org.apache.solr.common.util.StrUtils;
|
||||||
import org.apache.solr.util.TimeOut;
|
import org.apache.solr.util.TimeOut;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import static org.apache.solr.common.params.CommonParams.NAME;
|
||||||
|
|
||||||
|
|
||||||
public class CreateAliasCmd implements Cmd {
|
public class CreateAliasCmd implements Cmd {
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
@ -51,24 +52,12 @@ public class CreateAliasCmd implements Cmd {
|
||||||
public void call(ClusterState state, ZkNodeProps message, NamedList results)
|
public void call(ClusterState state, ZkNodeProps message, NamedList results)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
String aliasName = message.getStr(NAME);
|
String aliasName = message.getStr(NAME);
|
||||||
String collections = message.getStr("collections");
|
String collections = message.getStr("collections"); // could be comma delimited list
|
||||||
|
|
||||||
ZkStateReader zkStateReader = ocmh.zkStateReader;
|
ZkStateReader zkStateReader = ocmh.zkStateReader;
|
||||||
Map<String, String> prevColAliases = zkStateReader.getAliases().getCollectionAliasMap();
|
validateAllCollectionsExistAndNoDups(collections, zkStateReader);
|
||||||
validateAllCollectionsExist(collections, prevColAliases, zkStateReader.getClusterState());
|
|
||||||
|
|
||||||
Map<String, Map<String, String>> newAliasesMap = new HashMap<>();
|
byte[] jsonBytes = zkStateReader.getAliases().cloneWithCollectionAlias(aliasName, collections).toJSON();
|
||||||
Map<String, String> newCollectionAliasesMap = new HashMap<>();
|
|
||||||
if (prevColAliases != null) {
|
|
||||||
newCollectionAliasesMap.putAll(prevColAliases);
|
|
||||||
}
|
|
||||||
newCollectionAliasesMap.put(aliasName, collections);
|
|
||||||
newAliasesMap.put("collection", newCollectionAliasesMap);
|
|
||||||
Aliases newAliases = new Aliases(newAliasesMap);
|
|
||||||
byte[] jsonBytes = null;
|
|
||||||
if (newAliases.collectionAliasSize() > 0) { // only sub map right now
|
|
||||||
jsonBytes = Utils.toJSON(newAliases.getAliasMap());
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true);
|
zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true);
|
||||||
|
|
||||||
|
@ -84,10 +73,16 @@ public class CreateAliasCmd implements Cmd {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void validateAllCollectionsExist(String collections, Map<String,String> prevColAliases, ClusterState clusterState) {
|
private void validateAllCollectionsExistAndNoDups(String collections, ZkStateReader zkStateReader) {
|
||||||
String[] collectionArr = collections.split(",");
|
List<String> collectionArr = StrUtils.splitSmart(collections, ",", true);
|
||||||
for (String collection:collectionArr) {
|
if (new HashSet<>(collectionArr).size() != collectionArr.size()) {
|
||||||
if (clusterState.getCollectionOrNull(collection) == null && (prevColAliases == null || !prevColAliases.containsKey(collection))) {
|
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||||
|
String.format(Locale.ROOT, "Can't create collection alias for collections='%s', since it contains duplicates", collections));
|
||||||
|
}
|
||||||
|
ClusterState clusterState = zkStateReader.getClusterState();
|
||||||
|
Set<String> aliasNames = zkStateReader.getAliases().getCollectionAliasListMap().keySet();
|
||||||
|
for (String collection : collectionArr) {
|
||||||
|
if (clusterState.getCollectionOrNull(collection) == null && !aliasNames.contains(collection)) {
|
||||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||||
String.format(Locale.ROOT, "Can't create collection alias for collections='%s', '%s' is not an existing collection or alias", collections, collection));
|
String.format(Locale.ROOT, "Can't create collection alias for collections='%s', '%s' is not an existing collection or alias", collections, collection));
|
||||||
}
|
}
|
||||||
|
@ -95,14 +90,11 @@ public class CreateAliasCmd implements Cmd {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkForAlias(String name, String value) {
|
private void checkForAlias(String name, String value) {
|
||||||
|
|
||||||
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
|
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
Aliases aliases;
|
|
||||||
while (!timeout.hasTimedOut()) {
|
while (!timeout.hasTimedOut()) {
|
||||||
aliases = ocmh.zkStateReader.getAliases();
|
String collections = ocmh.zkStateReader.getAliases().getCollectionAliasMap().get(name);
|
||||||
String collections = aliases.getCollectionAlias(name);
|
if (Objects.equals(collections, value)) {
|
||||||
if (collections != null && collections.equals(value)) {
|
|
||||||
success = true;
|
success = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,17 +19,13 @@
|
||||||
package org.apache.solr.cloud;
|
package org.apache.solr.cloud;
|
||||||
|
|
||||||
import java.lang.invoke.MethodHandles;
|
import java.lang.invoke.MethodHandles;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.apache.solr.common.SolrException;
|
import org.apache.solr.common.SolrException;
|
||||||
import org.apache.solr.common.cloud.Aliases;
|
|
||||||
import org.apache.solr.common.cloud.ClusterState;
|
import org.apache.solr.common.cloud.ClusterState;
|
||||||
import org.apache.solr.common.cloud.ZkNodeProps;
|
import org.apache.solr.common.cloud.ZkNodeProps;
|
||||||
import org.apache.solr.common.cloud.ZkStateReader;
|
import org.apache.solr.common.cloud.ZkStateReader;
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
import org.apache.solr.common.util.Utils;
|
|
||||||
import org.apache.solr.util.TimeOut;
|
import org.apache.solr.util.TimeOut;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
|
@ -49,20 +45,11 @@ public class DeleteAliasCmd implements OverseerCollectionMessageHandler.Cmd {
|
||||||
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
|
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
|
||||||
String aliasName = message.getStr(NAME);
|
String aliasName = message.getStr(NAME);
|
||||||
|
|
||||||
Map<String,Map<String,String>> newAliasesMap = new HashMap<>();
|
|
||||||
Map<String,String> newCollectionAliasesMap = new HashMap<>();
|
|
||||||
ZkStateReader zkStateReader = ocmh.zkStateReader;
|
ZkStateReader zkStateReader = ocmh.zkStateReader;
|
||||||
newCollectionAliasesMap.putAll(zkStateReader.getAliases().getCollectionAliasMap());
|
byte[] jsonBytes = zkStateReader.getAliases().cloneWithCollectionAlias(aliasName, null).toJSON();
|
||||||
newCollectionAliasesMap.remove(aliasName);
|
|
||||||
newAliasesMap.put("collection", newCollectionAliasesMap);
|
|
||||||
Aliases newAliases = new Aliases(newAliasesMap);
|
|
||||||
byte[] jsonBytes = null;
|
|
||||||
if (newAliases.collectionAliasSize() > 0) { // only sub map right now
|
|
||||||
jsonBytes = Utils.toJSON(newAliases.getAliasMap());
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
zkStateReader.getZkClient().setData(ZkStateReader.ALIASES,
|
zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true);
|
||||||
jsonBytes, true);
|
|
||||||
checkForAliasAbsence(aliasName);
|
checkForAliasAbsence(aliasName);
|
||||||
// some fudge for other nodes
|
// some fudge for other nodes
|
||||||
Thread.sleep(100);
|
Thread.sleep(100);
|
||||||
|
@ -76,13 +63,10 @@ public class DeleteAliasCmd implements OverseerCollectionMessageHandler.Cmd {
|
||||||
|
|
||||||
}
|
}
|
||||||
private void checkForAliasAbsence(String name) {
|
private void checkForAliasAbsence(String name) {
|
||||||
|
|
||||||
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
|
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
Aliases aliases = null;
|
|
||||||
while (! timeout.hasTimedOut()) {
|
while (! timeout.hasTimedOut()) {
|
||||||
aliases = ocmh.zkStateReader.getAliases();
|
String collections = ocmh.zkStateReader.getAliases().getCollectionAliasMap().get(name);
|
||||||
String collections = aliases.getCollectionAlias(name);
|
|
||||||
if (collections == null) {
|
if (collections == null) {
|
||||||
success = true;
|
success = true;
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -38,20 +38,18 @@ import org.apache.solr.common.cloud.ZkStateReader;
|
||||||
import org.apache.solr.common.params.ShardParams;
|
import org.apache.solr.common.params.ShardParams;
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||||
import org.apache.solr.common.util.StrUtils;
|
|
||||||
import org.apache.solr.common.util.Utils;
|
import org.apache.solr.common.util.Utils;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
|
|
||||||
public class ClusterStatus {
|
public class ClusterStatus {
|
||||||
private final ZkStateReader zkStateReader;
|
private final ZkStateReader zkStateReader;
|
||||||
private final String collection;
|
private final ZkNodeProps message;
|
||||||
private ZkNodeProps message;
|
private final String collection; // maybe null
|
||||||
|
|
||||||
public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) {
|
public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) {
|
||||||
this.zkStateReader = zkStateReader;
|
this.zkStateReader = zkStateReader;
|
||||||
this.message = props;
|
this.message = props;
|
||||||
collection = props.getStr(ZkStateReader.COLLECTION_PROP);
|
collection = props.getStr(ZkStateReader.COLLECTION_PROP);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
|
@ -60,23 +58,17 @@ public class ClusterStatus {
|
||||||
// read aliases
|
// read aliases
|
||||||
Aliases aliases = zkStateReader.getAliases();
|
Aliases aliases = zkStateReader.getAliases();
|
||||||
Map<String, List<String>> collectionVsAliases = new HashMap<>();
|
Map<String, List<String>> collectionVsAliases = new HashMap<>();
|
||||||
Map<String, String> aliasVsCollections = aliases.getCollectionAliasMap();
|
Map<String, List<String>> aliasVsCollections = aliases.getCollectionAliasListMap();
|
||||||
if (aliasVsCollections != null) {
|
for (Map.Entry<String, List<String>> entry : aliasVsCollections.entrySet()) {
|
||||||
for (Map.Entry<String, String> entry : aliasVsCollections.entrySet()) {
|
|
||||||
List<String> colls = StrUtils.splitSmart(entry.getValue(), ',');
|
|
||||||
String alias = entry.getKey();
|
String alias = entry.getKey();
|
||||||
|
List<String> colls = entry.getValue();
|
||||||
for (String coll : colls) {
|
for (String coll : colls) {
|
||||||
if (collection == null || collection.equals(coll)) {
|
if (collection == null || collection.equals(coll)) {
|
||||||
List<String> list = collectionVsAliases.get(coll);
|
List<String> list = collectionVsAliases.computeIfAbsent(coll, k -> new ArrayList<>());
|
||||||
if (list == null) {
|
|
||||||
list = new ArrayList<>();
|
|
||||||
collectionVsAliases.put(coll, list);
|
|
||||||
}
|
|
||||||
list.add(alias);
|
list.add(alias);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Map roles = null;
|
Map roles = null;
|
||||||
if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {
|
if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {
|
||||||
|
@ -158,8 +150,9 @@ public class ClusterStatus {
|
||||||
}
|
}
|
||||||
|
|
||||||
// add the alias map too
|
// add the alias map too
|
||||||
if (aliasVsCollections != null && !aliasVsCollections.isEmpty()) {
|
Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap(); // comma delim
|
||||||
clusterStatus.add("aliases", aliasVsCollections);
|
if (!collectionAliasMap.isEmpty()) {
|
||||||
|
clusterStatus.add("aliases", collectionAliasMap);
|
||||||
}
|
}
|
||||||
|
|
||||||
// add the roles map
|
// add the roles map
|
||||||
|
@ -172,6 +165,7 @@ public class ClusterStatus {
|
||||||
|
|
||||||
results.add("cluster", clusterStatus);
|
results.add("cluster", clusterStatus);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get collection status from cluster state.
|
* Get collection status from cluster state.
|
||||||
* Can return collection status by given shard name.
|
* Can return collection status by given shard name.
|
||||||
|
|
|
@ -58,10 +58,8 @@ class SolrSchema extends AbstractSchema {
|
||||||
}
|
}
|
||||||
|
|
||||||
Aliases aliases = zkStateReader.getAliases();
|
Aliases aliases = zkStateReader.getAliases();
|
||||||
if(aliases.collectionAliasSize() > 0) {
|
for (String alias : aliases.getCollectionAliasListMap().keySet()) {
|
||||||
for (Map.Entry<String, String> alias : aliases.getCollectionAliasMap().entrySet()) {
|
builder.put(alias, new SolrTable(this, alias));
|
||||||
builder.put(alias.getKey(), new SolrTable(this, alias.getValue()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return builder.build();
|
return builder.build();
|
||||||
|
|
|
@ -17,7 +17,7 @@
|
||||||
package org.apache.solr.search.join;
|
package org.apache.solr.search.join;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Map;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
|
||||||
import org.apache.lucene.index.DocValuesType;
|
import org.apache.lucene.index.DocValuesType;
|
||||||
|
@ -275,10 +275,9 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
|
||||||
public static String getCoreName(final String fromIndex, CoreContainer container) {
|
public static String getCoreName(final String fromIndex, CoreContainer container) {
|
||||||
if (container.isZooKeeperAware()) {
|
if (container.isZooKeeperAware()) {
|
||||||
ZkController zkController = container.getZkController();
|
ZkController zkController = container.getZkController();
|
||||||
final String resolved =
|
final String resolved = resolveAlias(fromIndex, zkController);
|
||||||
zkController.getClusterState().hasCollection(fromIndex)
|
// TODO DWS: no need for this since later, clusterState.getCollection will throw a reasonable error
|
||||||
? fromIndex : resolveAlias(fromIndex, zkController);
|
if (!zkController.getClusterState().hasCollection(resolved)) {
|
||||||
if (resolved == null) {
|
|
||||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||||
"SolrCloud join: Collection '" + fromIndex + "' not found!");
|
"SolrCloud join: Collection '" + fromIndex + "' not found!");
|
||||||
}
|
}
|
||||||
|
@ -289,21 +288,14 @@ public class ScoreJoinQParserPlugin extends QParserPlugin {
|
||||||
|
|
||||||
private static String resolveAlias(String fromIndex, ZkController zkController) {
|
private static String resolveAlias(String fromIndex, ZkController zkController) {
|
||||||
final Aliases aliases = zkController.getZkStateReader().getAliases();
|
final Aliases aliases = zkController.getZkStateReader().getAliases();
|
||||||
if (aliases != null) {
|
List<String> collections = aliases.resolveAliases(fromIndex); // if not an alias, returns input
|
||||||
final String resolved;
|
if (collections.size() != 1) {
|
||||||
Map<String, String> collectionAliases = aliases.getCollectionAliasMap();
|
|
||||||
resolved = (collectionAliases != null) ? collectionAliases.get(fromIndex) : null;
|
|
||||||
if (resolved != null) {
|
|
||||||
if (resolved.split(",").length > 1) {
|
|
||||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||||
"SolrCloud join: Collection alias '" + fromIndex +
|
"SolrCloud join: Collection alias '" + fromIndex +
|
||||||
"' maps to multiple collections (" + resolved +
|
"' maps to multiple collections (" + collections +
|
||||||
"), which is not currently supported for joins.");
|
"), which is not currently supported for joins.");
|
||||||
}
|
}
|
||||||
return resolved;
|
return collections.get(0);
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) {
|
private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) {
|
||||||
|
|
|
@ -30,8 +30,8 @@ import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Enumeration;
|
import java.util.Enumeration;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
@ -72,7 +72,9 @@ import org.apache.solr.common.params.CommonParams;
|
||||||
import org.apache.solr.common.params.MapSolrParams;
|
import org.apache.solr.common.params.MapSolrParams;
|
||||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||||
import org.apache.solr.common.params.SolrParams;
|
import org.apache.solr.common.params.SolrParams;
|
||||||
|
import org.apache.solr.common.util.CommandOperation;
|
||||||
import org.apache.solr.common.util.ContentStream;
|
import org.apache.solr.common.util.ContentStream;
|
||||||
|
import org.apache.solr.common.util.JsonSchemaValidator;
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||||
import org.apache.solr.common.util.StrUtils;
|
import org.apache.solr.common.util.StrUtils;
|
||||||
|
@ -101,8 +103,6 @@ import org.apache.solr.servlet.SolrDispatchFilter.Action;
|
||||||
import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
|
import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
|
||||||
import org.apache.solr.servlet.cache.Method;
|
import org.apache.solr.servlet.cache.Method;
|
||||||
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
|
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
|
||||||
import org.apache.solr.common.util.CommandOperation;
|
|
||||||
import org.apache.solr.common.util.JsonSchemaValidator;
|
|
||||||
import org.apache.solr.util.RTimerTree;
|
import org.apache.solr.util.RTimerTree;
|
||||||
import org.apache.solr.util.TimeOut;
|
import org.apache.solr.util.TimeOut;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
|
@ -160,13 +160,10 @@ public class HttpSolrCall {
|
||||||
protected String coreUrl;
|
protected String coreUrl;
|
||||||
protected SolrConfig config;
|
protected SolrConfig config;
|
||||||
protected Map<String, Integer> invalidStates;
|
protected Map<String, Integer> invalidStates;
|
||||||
protected boolean usingAliases = false;
|
|
||||||
|
|
||||||
//The states of client that is invalid in this request
|
//The states of client that is invalid in this request
|
||||||
protected Aliases aliases = null;
|
protected String origCorename; // What's in the URL path; might reference a collection/alias or a Solr core name
|
||||||
protected String corename = "";
|
protected List<String> collectionsList; // The list of SolrCloud collections if in SolrCloud (usually 1)
|
||||||
protected String origCorename = null;
|
|
||||||
|
|
||||||
|
|
||||||
public RequestType getRequestType() {
|
public RequestType getRequestType() {
|
||||||
return requestType;
|
return requestType;
|
||||||
|
@ -174,13 +171,6 @@ public class HttpSolrCall {
|
||||||
|
|
||||||
protected RequestType requestType;
|
protected RequestType requestType;
|
||||||
|
|
||||||
|
|
||||||
public List<String> getCollectionsList() {
|
|
||||||
return collectionsList;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected List<String> collectionsList;
|
|
||||||
|
|
||||||
public HttpSolrCall(SolrDispatchFilter solrDispatchFilter, CoreContainer cores,
|
public HttpSolrCall(SolrDispatchFilter solrDispatchFilter, CoreContainer cores,
|
||||||
HttpServletRequest request, HttpServletResponse response, boolean retry) {
|
HttpServletRequest request, HttpServletResponse response, boolean retry) {
|
||||||
this.solrDispatchFilter = solrDispatchFilter;
|
this.solrDispatchFilter = solrDispatchFilter;
|
||||||
|
@ -206,7 +196,6 @@ public class HttpSolrCall {
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public HttpServletRequest getReq() {
|
public HttpServletRequest getReq() {
|
||||||
return req;
|
return req;
|
||||||
}
|
}
|
||||||
|
@ -219,12 +208,22 @@ public class HttpSolrCall {
|
||||||
return queryParams;
|
return queryParams;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected Aliases getAliases() {
|
||||||
|
return cores.isZooKeeperAware() ? cores.getZkController().getZkStateReader().getAliases() : Aliases.EMPTY;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The collection(s) referenced in this request. Populated in {@link #init()}. Not null. */
|
||||||
|
public List<String> getCollectionsList() {
|
||||||
|
return collectionsList != null ? collectionsList : Collections.emptyList();
|
||||||
|
}
|
||||||
|
|
||||||
protected void init() throws Exception {
|
protected void init() throws Exception {
|
||||||
// check for management path
|
// check for management path
|
||||||
String alternate = cores.getManagementPath();
|
String alternate = cores.getManagementPath();
|
||||||
if (alternate != null && path.startsWith(alternate)) {
|
if (alternate != null && path.startsWith(alternate)) {
|
||||||
path = path.substring(0, alternate.length());
|
path = path.substring(0, alternate.length());
|
||||||
}
|
}
|
||||||
|
|
||||||
// unused feature ?
|
// unused feature ?
|
||||||
int idx = path.indexOf(':');
|
int idx = path.indexOf(':');
|
||||||
if (idx > 0) {
|
if (idx > 0) {
|
||||||
|
@ -232,8 +231,6 @@ public class HttpSolrCall {
|
||||||
path = path.substring(0, idx);
|
path = path.substring(0, idx);
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean usingAliases = false;
|
|
||||||
|
|
||||||
// Check for container handlers
|
// Check for container handlers
|
||||||
handler = cores.getRequestHandler(path);
|
handler = cores.getRequestHandler(path);
|
||||||
if (handler != null) {
|
if (handler != null) {
|
||||||
|
@ -242,69 +239,59 @@ public class HttpSolrCall {
|
||||||
requestType = RequestType.ADMIN;
|
requestType = RequestType.ADMIN;
|
||||||
action = ADMIN;
|
action = ADMIN;
|
||||||
return;
|
return;
|
||||||
} else {
|
}
|
||||||
//otherwise, we should find a core from the path
|
|
||||||
|
// Parse a core or collection name from the path and attempt to see if it's a core name
|
||||||
idx = path.indexOf("/", 1);
|
idx = path.indexOf("/", 1);
|
||||||
if (idx > 1) {
|
if (idx > 1) {
|
||||||
// try to get the corename as a request parameter first
|
origCorename = path.substring(1, idx);
|
||||||
corename = path.substring(1, idx);
|
|
||||||
|
|
||||||
// look at aliases
|
// Try to resolve a Solr core name
|
||||||
if (cores.isZooKeeperAware()) {
|
core = cores.getCore(origCorename);
|
||||||
origCorename = corename;
|
|
||||||
ZkStateReader reader = cores.getZkController().getZkStateReader();
|
|
||||||
aliases = reader.getAliases();
|
|
||||||
if (aliases != null && aliases.collectionAliasSize() > 0) {
|
|
||||||
usingAliases = true;
|
|
||||||
String alias = aliases.getCollectionAlias(corename);
|
|
||||||
if (alias != null) {
|
|
||||||
collectionsList = StrUtils.splitSmart(alias, ",", true);
|
|
||||||
corename = collectionsList.get(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
core = cores.getCore(corename);
|
|
||||||
if (core != null) {
|
if (core != null) {
|
||||||
path = path.substring(idx);
|
path = path.substring(idx);
|
||||||
} else if (cores.isCoreLoading(corename)) { // extra mem barriers, so don't look at this before trying to get core
|
|
||||||
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
|
|
||||||
} else {
|
} else {
|
||||||
|
if (cores.isCoreLoading(origCorename)) { // extra mem barriers, so don't look at this before trying to get core
|
||||||
|
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
|
||||||
|
}
|
||||||
// the core may have just finished loading
|
// the core may have just finished loading
|
||||||
core = cores.getCore(corename);
|
core = cores.getCore(origCorename);
|
||||||
if (core != null) {
|
if (core != null) {
|
||||||
path = path.substring(idx);
|
path = path.substring(idx);
|
||||||
}
|
} else {
|
||||||
}
|
|
||||||
}
|
|
||||||
if (core == null) {
|
|
||||||
if (!cores.isZooKeeperAware()) {
|
if (!cores.isZooKeeperAware()) {
|
||||||
core = cores.getCore("");
|
core = cores.getCore("");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (core == null && cores.isZooKeeperAware()) {
|
|
||||||
// we couldn't find the core - lets make sure a collection was not specified instead
|
|
||||||
boolean isPreferLeader = false;
|
|
||||||
if (path.endsWith("/update") || path.contains("/update/")) {
|
|
||||||
isPreferLeader = true;
|
|
||||||
}
|
}
|
||||||
core = getCoreByCollection(corename, isPreferLeader);
|
|
||||||
|
if (cores.isZooKeeperAware()) {
|
||||||
|
// init collectionList (usually one name but not when there are aliases)
|
||||||
|
String def = core != null ? core.getCoreDescriptor().getCollectionName() : origCorename;
|
||||||
|
collectionsList = resolveCollectionListOrAlias(queryParams.get(COLLECTION_PROP, def)); // &collection= takes precedence
|
||||||
|
|
||||||
|
if (core == null) {
|
||||||
|
// lookup core from collection, or route away if need to
|
||||||
|
String collectionName = collectionsList.isEmpty() ? null : collectionsList.get(0); // route to 1st
|
||||||
|
//TODO try the other collections if can't find a local replica of the first? (and do to V2HttpSolrCall)
|
||||||
|
|
||||||
|
boolean isPreferLeader = (path.endsWith("/update") || path.contains("/update/"));
|
||||||
|
|
||||||
|
core = getCoreByCollection(collectionName, isPreferLeader); // find a local replica/core for the collection
|
||||||
if (core != null) {
|
if (core != null) {
|
||||||
// we found a core, update the path
|
if (idx > 0) {
|
||||||
path = path.substring(idx);
|
path = path.substring(idx);
|
||||||
if (collectionsList == null)
|
|
||||||
collectionsList = new ArrayList<>();
|
|
||||||
collectionsList.add(corename);
|
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
// if we couldn't find it locally, look on other nodes
|
// if we couldn't find it locally, look on other nodes
|
||||||
extractRemotePath(corename, origCorename, idx);
|
extractRemotePath(collectionName, origCorename, idx);
|
||||||
if (action != null) return;
|
if (action != null) return;
|
||||||
//core is not available locally or remotely
|
//core is not available locally or remotely
|
||||||
autoCreateSystemColl(corename);
|
autoCreateSystemColl(collectionName);
|
||||||
if(action != null) return;
|
if (action != null) return;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// With a valid core...
|
// With a valid core...
|
||||||
|
@ -314,7 +301,6 @@ public class HttpSolrCall {
|
||||||
// get or create/cache the parser for the core
|
// get or create/cache the parser for the core
|
||||||
SolrRequestParsers parser = config.getRequestParsers();
|
SolrRequestParsers parser = config.getRequestParsers();
|
||||||
|
|
||||||
|
|
||||||
// Determine the handler from the url path if not set
|
// Determine the handler from the url path if not set
|
||||||
// (we might already have selected the cores handler)
|
// (we might already have selected the cores handler)
|
||||||
extractHandlerFromURLPath(parser);
|
extractHandlerFromURLPath(parser);
|
||||||
|
@ -329,9 +315,7 @@ public class HttpSolrCall {
|
||||||
|
|
||||||
invalidStates = checkStateVersionsAreValid(solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
|
invalidStates = checkStateVersionsAreValid(solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
|
||||||
|
|
||||||
if (usingAliases) {
|
addCollectionParamIfNeeded(getCollectionsList());
|
||||||
processAliases(aliases, collectionsList);
|
|
||||||
}
|
|
||||||
|
|
||||||
action = PROCESS;
|
action = PROCESS;
|
||||||
return; // we are done with a valid handler
|
return; // we are done with a valid handler
|
||||||
|
@ -374,18 +358,24 @@ public class HttpSolrCall {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String lookupAliases(String collName) {
|
/**
|
||||||
ZkStateReader reader = cores.getZkController().getZkStateReader();
|
* Resolves the parameter as a potential comma delimited list of collections, and resolves aliases too.
|
||||||
aliases = reader.getAliases();
|
* One level of aliases pointing to another alias is supported.
|
||||||
if (aliases != null && aliases.collectionAliasSize() > 0) {
|
* De-duplicates and retains the order.
|
||||||
usingAliases = true;
|
* {@link #getCollectionsList()}
|
||||||
String alias = aliases.getCollectionAlias(collName);
|
*/
|
||||||
if (alias != null) {
|
protected List<String> resolveCollectionListOrAlias(String collectionStr) {
|
||||||
collectionsList = StrUtils.splitSmart(alias, ",", true);
|
if (collectionStr == null) {
|
||||||
return collectionsList.get(0);
|
return Collections.emptyList();
|
||||||
}
|
}
|
||||||
|
LinkedHashSet<String> resultList = new LinkedHashSet<>();
|
||||||
|
Aliases aliases = getAliases();
|
||||||
|
List<String> inputCollections = StrUtils.splitSmart(collectionStr, ",", true);
|
||||||
|
for (String inputCollection : inputCollections) {
|
||||||
|
List<String> resolvedCollections = aliases.resolveAliases(inputCollection);
|
||||||
|
resultList.addAll(resolvedCollections);
|
||||||
}
|
}
|
||||||
return null;
|
return new ArrayList<>(resultList);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -432,9 +422,9 @@ public class HttpSolrCall {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void extractRemotePath(String corename, String origCorename, int idx) throws UnsupportedEncodingException, KeeperException, InterruptedException {
|
protected void extractRemotePath(String collectionName, String origCorename, int idx) throws UnsupportedEncodingException, KeeperException, InterruptedException {
|
||||||
if (core == null && idx > 0) {
|
if (core == null && idx > 0) {
|
||||||
coreUrl = getRemotCoreUrl(corename, origCorename);
|
coreUrl = getRemotCoreUrl(collectionName, origCorename);
|
||||||
// don't proxy for internal update requests
|
// don't proxy for internal update requests
|
||||||
invalidStates = checkStateVersionsAreValid(queryParams.get(CloudSolrClient.STATE_VERSION));
|
invalidStates = checkStateVersionsAreValid(queryParams.get(CloudSolrClient.STATE_VERSION));
|
||||||
if (coreUrl != null
|
if (coreUrl != null
|
||||||
|
@ -745,41 +735,33 @@ public class HttpSolrCall {
|
||||||
handler.handleRequest(solrReq, solrResp);
|
handler.handleRequest(solrReq, solrResp);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void processAliases(Aliases aliases,
|
/**
|
||||||
List<String> collectionsList) {
|
* Sets the "collection" parameter on the request to the list of alias-resolved collections for this request.
|
||||||
String collection = solrReq.getParams().get(COLLECTION_PROP);
|
* It can be avoided sometimes.
|
||||||
if (collection != null) {
|
* Note: {@link org.apache.solr.handler.component.HttpShardHandler} processes this param.
|
||||||
collectionsList = StrUtils.splitSmart(collection, ",", true);
|
* @see #getCollectionsList()
|
||||||
|
*/
|
||||||
|
protected void addCollectionParamIfNeeded(List<String> collections) {
|
||||||
|
if (collections.isEmpty()) {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
if (collectionsList != null) {
|
assert cores.isZooKeeperAware();
|
||||||
Set<String> newCollectionsList = new HashSet<>(
|
String collectionParam = queryParams.get(COLLECTION_PROP);
|
||||||
collectionsList.size());
|
// if there is no existing collection param and the core we go to is for the expected collection,
|
||||||
for (String col : collectionsList) {
|
// then we needn't add a collection param
|
||||||
String al = aliases.getCollectionAlias(col);
|
if (collectionParam == null && // if collection param already exists, we may need to over-write it
|
||||||
if (al != null) {
|
core != null && collections.equals(Collections.singletonList(core.getCoreDescriptor().getCollectionName()))) {
|
||||||
List<String> aliasList = StrUtils.splitSmart(al, ",", true);
|
return;
|
||||||
newCollectionsList.addAll(aliasList);
|
|
||||||
} else {
|
|
||||||
newCollectionsList.add(col);
|
|
||||||
}
|
}
|
||||||
|
String newCollectionParam = StrUtils.join(collections, ',');
|
||||||
|
if (newCollectionParam.equals(collectionParam)) {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
if (newCollectionsList.size() > 0) {
|
// TODO add a SolrRequest.getModifiableParams ?
|
||||||
StringBuilder collectionString = new StringBuilder();
|
ModifiableSolrParams params = new ModifiableSolrParams(solrReq.getParams());
|
||||||
Iterator<String> it = newCollectionsList.iterator();
|
params.set(COLLECTION_PROP, newCollectionParam);
|
||||||
int sz = newCollectionsList.size();
|
|
||||||
for (int i = 0; i < sz; i++) {
|
|
||||||
collectionString.append(it.next());
|
|
||||||
if (i < newCollectionsList.size() - 1) {
|
|
||||||
collectionString.append(",");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ModifiableSolrParams params = new ModifiableSolrParams(
|
|
||||||
solrReq.getParams());
|
|
||||||
params.set(COLLECTION_PROP, collectionString.toString());
|
|
||||||
solrReq.setParams(params);
|
solrReq.setParams(params);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void writeResponse(SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, Method reqMethod)
|
private void writeResponse(SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, Method reqMethod)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
@ -916,9 +898,6 @@ public class HttpSolrCall {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (collectionsList == null)
|
|
||||||
collectionsList = new ArrayList<>();
|
|
||||||
|
|
||||||
collectionsList.add(collectionName);
|
collectionsList.add(collectionName);
|
||||||
String coreUrl = getCoreUrl(collectionName, origCorename, clusterState,
|
String coreUrl = getCoreUrl(collectionName, origCorename, clusterState,
|
||||||
slices, byCoreName, true);
|
slices, byCoreName, true);
|
||||||
|
@ -984,11 +963,9 @@ public class HttpSolrCall {
|
||||||
|
|
||||||
SolrParams params = getQueryParams();
|
SolrParams params = getQueryParams();
|
||||||
final ArrayList<CollectionRequest> collectionRequests = new ArrayList<>();
|
final ArrayList<CollectionRequest> collectionRequests = new ArrayList<>();
|
||||||
if (getCollectionsList() != null) {
|
|
||||||
for (String collection : getCollectionsList()) {
|
for (String collection : getCollectionsList()) {
|
||||||
collectionRequests.add(new CollectionRequest(collection));
|
collectionRequests.add(new CollectionRequest(collection));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Extract collection name from the params in case of a Collection Admin request
|
// Extract collection name from the params in case of a Collection Admin request
|
||||||
if (getPath().equals("/admin/collections")) {
|
if (getPath().equals("/admin/collections")) {
|
||||||
|
@ -1000,14 +977,6 @@ public class HttpSolrCall {
|
||||||
collectionRequests.add(new CollectionRequest(params.get(COLLECTION_PROP)));
|
collectionRequests.add(new CollectionRequest(params.get(COLLECTION_PROP)));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle the case when it's a /select request and collections are specified as a param
|
|
||||||
if(resource.equals("/select") && params.get("collection") != null) {
|
|
||||||
collectionRequests.clear();
|
|
||||||
for(String collection:params.get("collection").split(",")) {
|
|
||||||
collectionRequests.add(new CollectionRequest(collection));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Populate the request type if the request is select or update
|
// Populate the request type if the request is select or update
|
||||||
if(requestType == RequestType.UNKNOWN) {
|
if(requestType == RequestType.UNKNOWN) {
|
||||||
if(resource.startsWith("/select") || resource.startsWith("/get"))
|
if(resource.startsWith("/select") || resource.startsWith("/get"))
|
||||||
|
@ -1016,15 +985,6 @@ public class HttpSolrCall {
|
||||||
requestType = RequestType.WRITE;
|
requestType = RequestType.WRITE;
|
||||||
}
|
}
|
||||||
|
|
||||||
// There's no collection explicitly mentioned, let's try and extract it from the core if one exists for
|
|
||||||
// the purpose of processing this request.
|
|
||||||
if (getCore() != null && (getCollectionsList() == null || getCollectionsList().size() == 0)) {
|
|
||||||
collectionRequests.add(new CollectionRequest(getCore().getCoreDescriptor().getCollectionName()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (getQueryParams().get(COLLECTION_PROP) != null)
|
|
||||||
collectionRequests.add(new CollectionRequest(getQueryParams().get(COLLECTION_PROP)));
|
|
||||||
|
|
||||||
return new AuthorizationContext() {
|
return new AuthorizationContext() {
|
||||||
@Override
|
@Override
|
||||||
public SolrParams getParams() {
|
public SolrParams getParams() {
|
||||||
|
|
|
@ -16,7 +16,11 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.solr.cloud;
|
package org.apache.solr.cloud;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
import org.apache.solr.client.solrj.SolrQuery;
|
import org.apache.solr.client.solrj.SolrQuery;
|
||||||
|
import org.apache.solr.client.solrj.SolrServerException;
|
||||||
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
|
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
|
||||||
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||||
|
@ -24,6 +28,8 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||||
import org.apache.solr.client.solrj.request.UpdateRequest;
|
import org.apache.solr.client.solrj.request.UpdateRequest;
|
||||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||||
import org.apache.solr.common.SolrException;
|
import org.apache.solr.common.SolrException;
|
||||||
|
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||||
|
import org.apache.solr.common.params.SolrParams;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
@ -62,92 +68,50 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
|
||||||
new CollectionAdminRequest.ListAliases().process(cluster.getSolrClient()).getAliases().get("testalias"));
|
new CollectionAdminRequest.ListAliases().process(cluster.getSolrClient()).getAliases().get("testalias"));
|
||||||
|
|
||||||
// search for alias
|
// search for alias
|
||||||
QueryResponse res = cluster.getSolrClient().query("testalias", new SolrQuery("*:*"));
|
searchSeveralWays("testalias", new SolrQuery("*:*"), 3);
|
||||||
assertEquals(3, res.getResults().getNumFound());
|
|
||||||
|
|
||||||
// search for alias with random non cloud client
|
// Use a comma delimited list, one of which is an alias
|
||||||
JettySolrRunner jetty = cluster.getRandomJetty(random());
|
searchSeveralWays("testalias,collection2", new SolrQuery("*:*"), 5);
|
||||||
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
|
|
||||||
res = client.query(new SolrQuery("*:*"));
|
|
||||||
assertEquals(3, res.getResults().getNumFound());
|
|
||||||
}
|
|
||||||
|
|
||||||
// create alias, collection2 first because it's not on every node
|
// create alias, collection2 first because it's not on every node
|
||||||
CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient());
|
CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient());
|
||||||
|
|
||||||
// search with new cloud client
|
searchSeveralWays("testalias", new SolrQuery("*:*"), 5);
|
||||||
try (CloudSolrClient cloudSolrClient = getCloudSolrClient(cluster.getZkServer().getZkAddress(), random().nextBoolean())) {
|
|
||||||
cloudSolrClient.setParallelUpdates(random().nextBoolean());
|
|
||||||
SolrQuery query = new SolrQuery("*:*");
|
|
||||||
query.set("collection", "testalias");
|
|
||||||
res = cloudSolrClient.query(query);
|
|
||||||
assertEquals(5, res.getResults().getNumFound());
|
|
||||||
|
|
||||||
// Try with setDefaultCollection
|
|
||||||
query = new SolrQuery("*:*");
|
|
||||||
cloudSolrClient.setDefaultCollection("testalias");
|
|
||||||
res = cloudSolrClient.query(query);
|
|
||||||
assertEquals(5, res.getResults().getNumFound());
|
|
||||||
}
|
|
||||||
|
|
||||||
// search for alias with random non cloud client
|
|
||||||
jetty = cluster.getRandomJetty(random());
|
|
||||||
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
|
|
||||||
SolrQuery query = new SolrQuery("*:*");
|
|
||||||
query.set("collection", "testalias");
|
|
||||||
res = client.query(query);
|
|
||||||
assertEquals(5, res.getResults().getNumFound());
|
|
||||||
|
|
||||||
// now without collections param
|
|
||||||
query = new SolrQuery("*:*");
|
|
||||||
res = client.query(query);
|
|
||||||
assertEquals(5, res.getResults().getNumFound());
|
|
||||||
}
|
|
||||||
|
|
||||||
// update alias
|
// update alias
|
||||||
CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient());
|
CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient());
|
||||||
|
|
||||||
// search for alias
|
searchSeveralWays("testalias", new SolrQuery("*:*"), 2);
|
||||||
SolrQuery query = new SolrQuery("*:*");
|
|
||||||
query.set("collection", "testalias");
|
|
||||||
res = cluster.getSolrClient().query(query);
|
|
||||||
assertEquals(2, res.getResults().getNumFound());
|
|
||||||
|
|
||||||
// set alias to two collections
|
// set alias to two collections
|
||||||
CollectionAdminRequest.createAlias("testalias", "collection1,collection2").process(cluster.getSolrClient());
|
CollectionAdminRequest.createAlias("testalias", "collection1,collection2").process(cluster.getSolrClient());
|
||||||
|
searchSeveralWays("testalias", new SolrQuery("*:*"), 5);
|
||||||
|
|
||||||
query = new SolrQuery("*:*");
|
// alias pointing to alias (one level of indirection is supported; more than that is not (may or may not work)
|
||||||
query.set("collection", "testalias");
|
// TODO dubious; remove?
|
||||||
res = cluster.getSolrClient().query(query);
|
CollectionAdminRequest.createAlias("testalias2", "testalias").process(cluster.getSolrClient());
|
||||||
assertEquals(5, res.getResults().getNumFound());
|
searchSeveralWays("testalias2", new SolrQuery("*:*"), 5);
|
||||||
|
|
||||||
// try a std client
|
|
||||||
// search 1 and 2, but have no collections param
|
|
||||||
query = new SolrQuery("*:*");
|
|
||||||
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
|
|
||||||
res = client.query(query);
|
|
||||||
assertEquals(5, res.getResults().getNumFound());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
// Test 2 aliases pointing to the same collection
|
||||||
CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient());
|
CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient());
|
||||||
|
|
||||||
// a second alias
|
|
||||||
CollectionAdminRequest.createAlias("testalias2", "collection2").process(cluster.getSolrClient());
|
CollectionAdminRequest.createAlias("testalias2", "collection2").process(cluster.getSolrClient());
|
||||||
|
|
||||||
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
|
// add one document to testalias, thus to collection2
|
||||||
new UpdateRequest()
|
new UpdateRequest()
|
||||||
.add("id", "11", "a_t", "humpty dumpy4 sat on a walls")
|
.add("id", "11", "a_t", "humpty dumpy4 sat on a walls")
|
||||||
.commit(cluster.getSolrClient(), "testalias");
|
.commit(cluster.getSolrClient(), "testalias"); // thus gets added to collection2
|
||||||
res = client.query(query);
|
|
||||||
assertEquals(3, res.getResults().getNumFound());
|
searchSeveralWays("testalias", new SolrQuery("*:*"), 3);
|
||||||
}
|
|
||||||
|
|
||||||
CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient());
|
CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient());
|
||||||
|
|
||||||
query = new SolrQuery("*:*");
|
searchSeveralWays("testalias", new SolrQuery("*:*"), 6);
|
||||||
query.set("collection", "testalias");
|
|
||||||
res = cluster.getSolrClient().query(query);
|
// add one document to testalias, which will route to collection2 because it's the first
|
||||||
assertEquals(6, res.getResults().getNumFound());
|
new UpdateRequest()
|
||||||
|
.add("id", "12", "a_t", "humpty dumpy5 sat on a walls")
|
||||||
|
.commit(cluster.getSolrClient(), "testalias"); // thus gets added to collection2
|
||||||
|
searchSeveralWays("collection2", new SolrQuery("*:*"), 4);
|
||||||
|
|
||||||
CollectionAdminRequest.deleteAlias("testalias").process(cluster.getSolrClient());
|
CollectionAdminRequest.deleteAlias("testalias").process(cluster.getSolrClient());
|
||||||
CollectionAdminRequest.deleteAlias("testalias2").process(cluster.getSolrClient());
|
CollectionAdminRequest.deleteAlias("testalias2").process(cluster.getSolrClient());
|
||||||
|
@ -160,6 +124,49 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
|
||||||
assertTrue("Unexpected exception message: " + e.getMessage(), e.getMessage().contains("Collection not found: testalias"));
|
assertTrue("Unexpected exception message: " + e.getMessage(), e.getMessage().contains("Collection not found: testalias"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void searchSeveralWays(String collectionList, SolrParams solrQuery, int expectedNumFound) throws IOException, SolrServerException {
|
||||||
|
searchSeveralWays(collectionList, solrQuery, res -> assertEquals(expectedNumFound, res.getResults().getNumFound()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void searchSeveralWays(String collectionList, SolrParams solrQuery, Consumer<QueryResponse> responseConsumer) throws IOException, SolrServerException {
|
||||||
|
if (random().nextBoolean()) {
|
||||||
|
// cluster's CloudSolrClient
|
||||||
|
responseConsumer.accept(cluster.getSolrClient().query(collectionList, solrQuery));
|
||||||
|
} else {
|
||||||
|
// new CloudSolrClient (random shardLeadersOnly)
|
||||||
|
try (CloudSolrClient solrClient = getCloudSolrClient(cluster)) {
|
||||||
|
if (random().nextBoolean()) {
|
||||||
|
solrClient.setDefaultCollection(collectionList);
|
||||||
|
responseConsumer.accept(solrClient.query(null, solrQuery));
|
||||||
|
} else {
|
||||||
|
responseConsumer.accept(solrClient.query(collectionList, solrQuery));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// HttpSolrClient
|
||||||
|
JettySolrRunner jetty = cluster.getRandomJetty(random());
|
||||||
|
if (random().nextBoolean()) {
|
||||||
|
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/" + collectionList)) {
|
||||||
|
responseConsumer.accept(client.query(null, solrQuery));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString())) {
|
||||||
|
responseConsumer.accept(client.query(collectionList, solrQuery));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recursively do again; this time with the &collection= param
|
||||||
|
if (solrQuery.get("collection") == null) {
|
||||||
|
// put in "collection" param
|
||||||
|
ModifiableSolrParams newParams = new ModifiableSolrParams(solrQuery);
|
||||||
|
newParams.set("collection", collectionList);
|
||||||
|
String maskedColl = new String[]{null, "bogus", "collection2", "collection1"}[random().nextInt(4)];
|
||||||
|
searchSeveralWays(maskedColl, newParams, responseConsumer);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
public void testErrorChecks() throws Exception {
|
public void testErrorChecks() throws Exception {
|
||||||
|
|
||||||
CollectionAdminRequest.createCollection("testErrorChecks-collection", "conf", 2, 1).process(cluster.getSolrClient());
|
CollectionAdminRequest.createCollection("testErrorChecks-collection", "conf", 2, 1).process(cluster.getSolrClient());
|
||||||
|
@ -189,6 +196,7 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
|
||||||
|
|
||||||
// Valid
|
// Valid
|
||||||
CollectionAdminRequest.createAlias("testalias", "testErrorChecks-collection").process(cluster.getSolrClient());
|
CollectionAdminRequest.createAlias("testalias", "testErrorChecks-collection").process(cluster.getSolrClient());
|
||||||
|
// TODO dubious; remove?
|
||||||
CollectionAdminRequest.createAlias("testalias2", "testalias").process(cluster.getSolrClient());
|
CollectionAdminRequest.createAlias("testalias2", "testalias").process(cluster.getSolrClient());
|
||||||
|
|
||||||
// Alias + invalid
|
// Alias + invalid
|
||||||
|
|
|
@ -112,13 +112,13 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{
|
||||||
@Test
|
@Test
|
||||||
public void testScore() throws Exception {
|
public void testScore() throws Exception {
|
||||||
//without score
|
//without score
|
||||||
testJoins(toColl, fromColl, toDocId, false);
|
testJoins(toColl, fromColl, toDocId, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testNoScore() throws Exception {
|
public void testNoScore() throws Exception {
|
||||||
//with score
|
//with score
|
||||||
testJoins(toColl, fromColl, toDocId, true);
|
testJoins(toColl, fromColl, toDocId, false);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -141,7 +141,7 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testJoins(String toColl, String fromColl, String toDocId, boolean isScoresTest)
|
private void testJoins(String toColl, String fromColl, String toDocId, boolean isScoresTest)
|
||||||
throws SolrServerException, IOException {
|
throws SolrServerException, IOException, InterruptedException {
|
||||||
// verify the join with fromIndex works
|
// verify the join with fromIndex works
|
||||||
final String fromQ = "match_s:c^2";
|
final String fromQ = "match_s:c^2";
|
||||||
CloudSolrClient client = cluster.getSolrClient();
|
CloudSolrClient client = cluster.getSolrClient();
|
||||||
|
@ -164,8 +164,7 @@ public class DistribJoinFromCollectionTest extends SolrCloudTestCase{
|
||||||
|
|
||||||
// create an alias for the fromIndex and then query through the alias
|
// create an alias for the fromIndex and then query through the alias
|
||||||
String alias = fromColl+"Alias";
|
String alias = fromColl+"Alias";
|
||||||
CollectionAdminRequest.CreateAlias request = CollectionAdminRequest.createAlias(alias,fromColl);
|
CollectionAdminRequest.createAlias(alias, fromColl).process(client);
|
||||||
request.process(client);
|
|
||||||
|
|
||||||
{
|
{
|
||||||
final String joinQ = "{!join " + anyScoreMode(isScoresTest)
|
final String joinQ = "{!join " + anyScoreMode(isScoresTest)
|
||||||
|
|
|
@ -18,6 +18,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
|
@ -53,20 +54,11 @@ public class DelegatingClusterStateProvider implements ClusterStateProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getAlias(String alias) {
|
public List<String> resolveAlias(String alias) {
|
||||||
if (delegate != null) {
|
if (delegate != null) {
|
||||||
return delegate.getAlias(alias);
|
return delegate.resolveAlias(alias);
|
||||||
} else {
|
} else {
|
||||||
return null;
|
return Collections.singletonList(alias);
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCollectionName(String name) {
|
|
||||||
if (delegate != null) {
|
|
||||||
return delegate.getCollectionName(name);
|
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -27,6 +27,7 @@ import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
@ -84,8 +85,8 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.slf4j.MDC;
|
import org.slf4j.MDC;
|
||||||
|
|
||||||
import static org.apache.solr.common.params.CommonParams.ID;
|
|
||||||
import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS;
|
import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS;
|
||||||
|
import static org.apache.solr.common.params.CommonParams.ID;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* SolrJ client class to communicate with SolrCloud.
|
* SolrJ client class to communicate with SolrCloud.
|
||||||
|
@ -204,6 +205,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
protected final StateCache collectionStateCache = new StateCache();
|
protected final StateCache collectionStateCache = new StateCache();
|
||||||
|
|
||||||
class ExpiringCachedDocCollection {
|
class ExpiringCachedDocCollection {
|
||||||
final DocCollection cached;
|
final DocCollection cached;
|
||||||
final long cachedAt;
|
final long cachedAt;
|
||||||
|
@ -222,7 +224,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
> TimeUnit.NANOSECONDS.convert(timeToLiveMs, TimeUnit.MILLISECONDS);
|
> TimeUnit.NANOSECONDS.convert(timeToLiveMs, TimeUnit.MILLISECONDS);
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean shoulRetry() {
|
boolean shouldRetry() {
|
||||||
if (maybeStale) {// we are not sure if it is stale so check with retry time
|
if (maybeStale) {// we are not sure if it is stale so check with retry time
|
||||||
if ((retriedAt == -1 ||
|
if ((retriedAt == -1 ||
|
||||||
(System.nanoTime() - retriedAt) > retryExpiryTime)) {
|
(System.nanoTime() - retriedAt) > retryExpiryTime)) {
|
||||||
|
@ -292,7 +294,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**Sets the cache ttl for DocCollection Objects cached . This is only applicable for collections which are persisted outside of clusterstate.json
|
/** Sets the cache ttl for DocCollection Objects cached . This is only applicable for collections which are persisted outside of clusterstate.json
|
||||||
* @param seconds ttl value in seconds
|
* @param seconds ttl value in seconds
|
||||||
*/
|
*/
|
||||||
public void setCollectionCacheTTl(int seconds){
|
public void setCollectionCacheTTl(int seconds){
|
||||||
|
@ -455,7 +457,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
|
|
||||||
private NamedList<Object> directUpdate(AbstractUpdateRequest request, String collection) throws SolrServerException {
|
private NamedList<Object> directUpdate(AbstractUpdateRequest request, String collection) throws SolrServerException {
|
||||||
UpdateRequest updateRequest = (UpdateRequest) request;
|
UpdateRequest updateRequest = (UpdateRequest) request;
|
||||||
ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
|
SolrParams params = request.getParams();
|
||||||
ModifiableSolrParams routableParams = new ModifiableSolrParams();
|
ModifiableSolrParams routableParams = new ModifiableSolrParams();
|
||||||
ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
|
ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
|
||||||
|
|
||||||
|
@ -471,9 +473,9 @@ public class CloudSolrClient extends SolrClient {
|
||||||
throw new SolrServerException("No collection param specified on request and no default collection has been set.");
|
throw new SolrServerException("No collection param specified on request and no default collection has been set.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
//Check to see if the collection is an alias.
|
//Check to see if the collection is an alias.
|
||||||
collection = stateProvider.getCollectionName(collection);
|
List<String> aliasedCollections = stateProvider.resolveAlias(collection);
|
||||||
|
collection = aliasedCollections.get(0); // pick 1st (consistent with HttpSolrCall behavior)
|
||||||
|
|
||||||
DocCollection col = getDocCollection(collection, null);
|
DocCollection col = getDocCollection(collection, null);
|
||||||
|
|
||||||
|
@ -786,11 +788,16 @@ public class CloudSolrClient extends SolrClient {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException {
|
public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException {
|
||||||
if (collection == null) {
|
// the collection parameter of the request overrides that of the parameter to this method
|
||||||
collection = request.getCollection();
|
String requestCollection = request.getCollection();
|
||||||
if (collection == null) collection = defaultCollection;
|
if (requestCollection != null) {
|
||||||
|
collection = requestCollection;
|
||||||
|
} else if (collection == null) {
|
||||||
|
collection = defaultCollection;
|
||||||
}
|
}
|
||||||
return requestWithRetryOnStaleState(request, 0, collection);
|
List<String> inputCollections =
|
||||||
|
collection == null ? Collections.emptyList() : StrUtils.splitSmart(collection, ",", true);
|
||||||
|
return requestWithRetryOnStaleState(request, 0, inputCollections);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -798,10 +805,8 @@ public class CloudSolrClient extends SolrClient {
|
||||||
* there's a chance that the request will fail due to cached stale state,
|
* there's a chance that the request will fail due to cached stale state,
|
||||||
* which means the state must be refreshed from ZK and retried.
|
* which means the state must be refreshed from ZK and retried.
|
||||||
*/
|
*/
|
||||||
protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, String collection)
|
protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, List<String> inputCollections)
|
||||||
throws SolrServerException, IOException {
|
throws SolrServerException, IOException {
|
||||||
SolrRequest originalRequest = request;
|
|
||||||
|
|
||||||
connect(); // important to call this before you start working with the ZkStateReader
|
connect(); // important to call this before you start working with the ZkStateReader
|
||||||
|
|
||||||
// build up a _stateVer_ param to pass to the server containing all of the
|
// build up a _stateVer_ param to pass to the server containing all of the
|
||||||
|
@ -818,8 +823,8 @@ public class CloudSolrClient extends SolrClient {
|
||||||
isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest();
|
isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest();
|
||||||
}
|
}
|
||||||
boolean isAdmin = ADMIN_PATHS.contains(request.getPath());
|
boolean isAdmin = ADMIN_PATHS.contains(request.getPath());
|
||||||
if (collection != null && !isAdmin && !isCollectionRequestOfV2) { // don't do _stateVer_ checking for admin, v2 api requests
|
if (!inputCollections.isEmpty() && !isAdmin && !isCollectionRequestOfV2) { // don't do _stateVer_ checking for admin, v2 api requests
|
||||||
Set<String> requestedCollectionNames = getCollectionNames(collection);
|
Set<String> requestedCollectionNames = resolveAliases(inputCollections);
|
||||||
|
|
||||||
StringBuilder stateVerParamBuilder = null;
|
StringBuilder stateVerParamBuilder = null;
|
||||||
for (String requestedCollection : requestedCollectionNames) {
|
for (String requestedCollection : requestedCollectionNames) {
|
||||||
|
@ -859,7 +864,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
|
|
||||||
NamedList<Object> resp = null;
|
NamedList<Object> resp = null;
|
||||||
try {
|
try {
|
||||||
resp = sendRequest(request, collection);
|
resp = sendRequest(request, inputCollections);
|
||||||
//to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there
|
//to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there
|
||||||
Object o = resp == null || resp.size() == 0 ? null : resp.get(STATE_VERSION, resp.size() - 1);
|
Object o = resp == null || resp.size() == 0 ? null : resp.get(STATE_VERSION, resp.size() - 1);
|
||||||
if(o != null && o instanceof Map) {
|
if(o != null && o instanceof Map) {
|
||||||
|
@ -878,7 +883,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
// don't do retry support for admin requests
|
// don't do retry support for admin requests
|
||||||
// or if the request doesn't have a collection specified
|
// or if the request doesn't have a collection specified
|
||||||
// or request is v2 api and its method is not GET
|
// or request is v2 api and its method is not GET
|
||||||
if (collection == null || isAdmin || (request instanceof V2Request && request.getMethod() != SolrRequest.METHOD.GET)) {
|
if (inputCollections.isEmpty() || isAdmin || (request instanceof V2Request && request.getMethod() != SolrRequest.METHOD.GET)) {
|
||||||
if (exc instanceof SolrServerException) {
|
if (exc instanceof SolrServerException) {
|
||||||
throw (SolrServerException)exc;
|
throw (SolrServerException)exc;
|
||||||
} else if (exc instanceof IOException) {
|
} else if (exc instanceof IOException) {
|
||||||
|
@ -894,8 +899,8 @@ public class CloudSolrClient extends SolrClient {
|
||||||
int errorCode = (rootCause instanceof SolrException) ?
|
int errorCode = (rootCause instanceof SolrException) ?
|
||||||
((SolrException)rootCause).code() : SolrException.ErrorCode.UNKNOWN.code;
|
((SolrException)rootCause).code() : SolrException.ErrorCode.UNKNOWN.code;
|
||||||
|
|
||||||
log.error("Request to collection {} failed due to ("+errorCode+
|
log.error("Request to collection {} failed due to (" + errorCode + ") {}, retry? " + retryCount,
|
||||||
") {}, retry? "+retryCount, collection, rootCause.toString());
|
inputCollections, rootCause.toString());
|
||||||
|
|
||||||
boolean wasCommError =
|
boolean wasCommError =
|
||||||
(rootCause instanceof ConnectException ||
|
(rootCause instanceof ConnectException ||
|
||||||
|
@ -907,7 +912,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
// it was a communication error. it is likely that
|
// it was a communication error. it is likely that
|
||||||
// the node to which the request to be sent is down . So , expire the state
|
// the node to which the request to be sent is down . So , expire the state
|
||||||
// so that the next attempt would fetch the fresh state
|
// so that the next attempt would fetch the fresh state
|
||||||
// just re-read state for all of them, if it has not been retired
|
// just re-read state for all of them, if it has not been retried
|
||||||
// in retryExpiryTime time
|
// in retryExpiryTime time
|
||||||
for (DocCollection ext : requestedCollections) {
|
for (DocCollection ext : requestedCollections) {
|
||||||
ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(ext.getName());
|
ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(ext.getName());
|
||||||
|
@ -919,7 +924,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
// and we could not get any information from the server
|
// and we could not get any information from the server
|
||||||
//it is probably not worth trying again and again because
|
//it is probably not worth trying again and again because
|
||||||
// the state would not have been updated
|
// the state would not have been updated
|
||||||
return requestWithRetryOnStaleState(request, retryCount + 1, collection);
|
return requestWithRetryOnStaleState(request, retryCount + 1, inputCollections);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -963,8 +968,8 @@ public class CloudSolrClient extends SolrClient {
|
||||||
|
|
||||||
// if the state was stale, then we retry the request once with new state pulled from Zk
|
// if the state was stale, then we retry the request once with new state pulled from Zk
|
||||||
if (stateWasStale) {
|
if (stateWasStale) {
|
||||||
log.warn("Re-trying request to collection(s) "+collection+" after stale state error from server.");
|
log.warn("Re-trying request to collection(s) "+inputCollections+" after stale state error from server.");
|
||||||
resp = requestWithRetryOnStaleState(request, retryCount+1, collection);
|
resp = requestWithRetryOnStaleState(request, retryCount+1, inputCollections);
|
||||||
} else {
|
} else {
|
||||||
if(exc instanceof SolrException) {
|
if(exc instanceof SolrException) {
|
||||||
throw exc;
|
throw exc;
|
||||||
|
@ -981,137 +986,97 @@ public class CloudSolrClient extends SolrClient {
|
||||||
return resp;
|
return resp;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected NamedList<Object> sendRequest(SolrRequest request, String collection)
|
protected NamedList<Object> sendRequest(SolrRequest request, List<String> inputCollections)
|
||||||
throws SolrServerException, IOException {
|
throws SolrServerException, IOException {
|
||||||
connect();
|
connect();
|
||||||
|
|
||||||
boolean sendToLeaders = false;
|
boolean sendToLeaders = false;
|
||||||
List<String> replicas = null;
|
|
||||||
|
|
||||||
if (request instanceof IsUpdateRequest) {
|
if (request instanceof IsUpdateRequest) {
|
||||||
if (request instanceof UpdateRequest) {
|
if (request instanceof UpdateRequest) {
|
||||||
|
String collection = inputCollections.isEmpty() ? null : inputCollections.get(0); // getting first mimics HttpSolrCall
|
||||||
NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection);
|
NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection);
|
||||||
if (response != null) {
|
if (response != null) {
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sendToLeaders = true;
|
sendToLeaders = true;
|
||||||
replicas = new ArrayList<>();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
SolrParams reqParams = request.getParams();
|
SolrParams reqParams = request.getParams();
|
||||||
if (reqParams == null) {
|
if (reqParams == null) { // TODO fix getParams to never return null!
|
||||||
reqParams = new ModifiableSolrParams();
|
reqParams = new ModifiableSolrParams();
|
||||||
}
|
}
|
||||||
List<String> theUrlList = new ArrayList<>();
|
|
||||||
|
final Set<String> liveNodes = stateProvider.getLiveNodes();
|
||||||
|
|
||||||
|
final List<String> theUrlList = new ArrayList<>(); // we populate this as follows...
|
||||||
|
|
||||||
if (request instanceof V2Request) {
|
if (request instanceof V2Request) {
|
||||||
Set<String> liveNodes = stateProvider.getLiveNodes();
|
|
||||||
if (!liveNodes.isEmpty()) {
|
if (!liveNodes.isEmpty()) {
|
||||||
List<String> liveNodesList = new ArrayList<>(liveNodes);
|
List<String> liveNodesList = new ArrayList<>(liveNodes);
|
||||||
Collections.shuffle(liveNodesList, rand);
|
Collections.shuffle(liveNodesList, rand);
|
||||||
theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0),
|
theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0),
|
||||||
(String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
|
(String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
|
||||||
}
|
}
|
||||||
|
|
||||||
} else if (ADMIN_PATHS.contains(request.getPath())) {
|
} else if (ADMIN_PATHS.contains(request.getPath())) {
|
||||||
Set<String> liveNodes = stateProvider.getLiveNodes();
|
|
||||||
for (String liveNode : liveNodes) {
|
for (String liveNode : liveNodes) {
|
||||||
theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode,
|
theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode,
|
||||||
(String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
|
(String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
|
|
||||||
if (collection == null) {
|
} else { // Typical...
|
||||||
throw new SolrServerException(
|
Set<String> collectionNames = resolveAliases(inputCollections);
|
||||||
"No collection param specified on request and no default collection has been set.");
|
if (collectionNames.isEmpty()) {
|
||||||
}
|
|
||||||
|
|
||||||
Set<String> collectionNames = getCollectionNames(collection);
|
|
||||||
if (collectionNames.size() == 0) {
|
|
||||||
throw new SolrException(ErrorCode.BAD_REQUEST,
|
throw new SolrException(ErrorCode.BAD_REQUEST,
|
||||||
"Could not find collection: " + collection);
|
"No collection param specified on request and no default collection has been set: " + inputCollections);
|
||||||
}
|
}
|
||||||
|
|
||||||
String shardKeys = reqParams.get(ShardParams._ROUTE_);
|
|
||||||
|
|
||||||
// TODO: not a big deal because of the caching, but we could avoid looking
|
// TODO: not a big deal because of the caching, but we could avoid looking
|
||||||
// at every shard
|
// at every shard when getting leaders if we tweaked some things
|
||||||
// when getting leaders if we tweaked some things
|
|
||||||
|
|
||||||
// Retrieve slices from the cloud state and, for each collection
|
// Retrieve slices from the cloud state and, for each collection specified, add it to the Map of slices.
|
||||||
// specified,
|
|
||||||
// add it to the Map of slices.
|
|
||||||
Map<String,Slice> slices = new HashMap<>();
|
Map<String,Slice> slices = new HashMap<>();
|
||||||
|
String shardKeys = reqParams.get(ShardParams._ROUTE_);
|
||||||
for (String collectionName : collectionNames) {
|
for (String collectionName : collectionNames) {
|
||||||
DocCollection col = getDocCollection(collectionName, null);
|
DocCollection col = getDocCollection(collectionName, null);
|
||||||
Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams , col);
|
Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams , col);
|
||||||
ClientUtils.addSlices(slices, collectionName, routeSlices, true);
|
ClientUtils.addSlices(slices, collectionName, routeSlices, true);
|
||||||
}
|
}
|
||||||
Set<String> liveNodes = stateProvider.getLiveNodes();
|
|
||||||
|
|
||||||
List<String> leaderUrlList = null;
|
// Gather URLs, grouped by leader or replica
|
||||||
List<String> urlList = null;
|
|
||||||
List<String> replicasList = null;
|
|
||||||
|
|
||||||
// build a map of unique nodes
|
|
||||||
// TODO: allow filtering by group, role, etc
|
// TODO: allow filtering by group, role, etc
|
||||||
Map<String,ZkNodeProps> nodes = new HashMap<>();
|
Set<String> seenNodes = new HashSet<>();
|
||||||
List<String> urlList2 = new ArrayList<>();
|
List<String> replicas = new ArrayList<>();
|
||||||
|
String joinedInputCollections = StrUtils.join(inputCollections, ',');
|
||||||
for (Slice slice : slices.values()) {
|
for (Slice slice : slices.values()) {
|
||||||
for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
|
for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
|
||||||
ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
|
ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
|
||||||
String node = coreNodeProps.getNodeName();
|
String node = coreNodeProps.getNodeName();
|
||||||
if (!liveNodes.contains(coreNodeProps.getNodeName())
|
if (!liveNodes.contains(node) // Must be a live node to continue
|
||||||
|| Replica.State.getState(coreNodeProps.getState()) != Replica.State.ACTIVE) continue;
|
|| Replica.State.getState(coreNodeProps.getState()) != Replica.State.ACTIVE) // Must be an ACTIVE replica to continue
|
||||||
if (nodes.put(node, nodeProps) == null) {
|
continue;
|
||||||
if (!sendToLeaders || coreNodeProps.isLeader()) {
|
if (seenNodes.add(node)) { // if we haven't yet collected a URL to this node...
|
||||||
String url;
|
String url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), joinedInputCollections);
|
||||||
if (reqParams.get(UpdateParams.COLLECTION) == null) {
|
if (sendToLeaders && coreNodeProps.isLeader()) {
|
||||||
url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
|
theUrlList.add(url); // put leaders here eagerly (if sendToLeader mode)
|
||||||
} else {
|
} else {
|
||||||
url = coreNodeProps.getCoreUrl();
|
replicas.add(url); // replicas here
|
||||||
}
|
|
||||||
urlList2.add(url);
|
|
||||||
} else {
|
|
||||||
String url;
|
|
||||||
if (reqParams.get(UpdateParams.COLLECTION) == null) {
|
|
||||||
url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
|
|
||||||
} else {
|
|
||||||
url = coreNodeProps.getCoreUrl();
|
|
||||||
}
|
|
||||||
replicas.add(url);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sendToLeaders) {
|
// Shuffle the leaders, if any (none if !sendToLeaders)
|
||||||
leaderUrlList = urlList2;
|
|
||||||
replicasList = replicas;
|
|
||||||
} else {
|
|
||||||
urlList = urlList2;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (sendToLeaders) {
|
|
||||||
theUrlList = new ArrayList<>(leaderUrlList.size());
|
|
||||||
theUrlList.addAll(leaderUrlList);
|
|
||||||
} else {
|
|
||||||
theUrlList = new ArrayList<>(urlList.size());
|
|
||||||
theUrlList.addAll(urlList);
|
|
||||||
}
|
|
||||||
|
|
||||||
Collections.shuffle(theUrlList, rand);
|
Collections.shuffle(theUrlList, rand);
|
||||||
if (sendToLeaders) {
|
|
||||||
ArrayList<String> theReplicas = new ArrayList<>(
|
// Shuffle the replicas, if any, and append to our list
|
||||||
replicasList.size());
|
Collections.shuffle(replicas, rand);
|
||||||
theReplicas.addAll(replicasList);
|
theUrlList.addAll(replicas);
|
||||||
Collections.shuffle(theReplicas, rand);
|
|
||||||
theUrlList.addAll(theReplicas);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (theUrlList.isEmpty()) {
|
if (theUrlList.isEmpty()) {
|
||||||
for (String s : collectionNames) {
|
collectionStateCache.keySet().removeAll(collectionNames);
|
||||||
if (s != null) collectionStateCache.remove(s);
|
|
||||||
}
|
|
||||||
throw new SolrException(SolrException.ErrorCode.INVALID_STATE,
|
throw new SolrException(SolrException.ErrorCode.INVALID_STATE,
|
||||||
"Could not find a healthy node to handle the request.");
|
"Could not find a healthy node to handle the request.");
|
||||||
}
|
}
|
||||||
|
@ -1122,24 +1087,20 @@ public class CloudSolrClient extends SolrClient {
|
||||||
return rsp.getResponse();
|
return rsp.getResponse();
|
||||||
}
|
}
|
||||||
|
|
||||||
private Set<String> getCollectionNames(String collection) {
|
/** Resolves the input collections to their possible aliased collections. Doesn't validate collection existence. */
|
||||||
// Extract each comma separated collection name and store in a List.
|
private LinkedHashSet<String> resolveAliases(List<String> inputCollections) {
|
||||||
List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true);
|
LinkedHashSet<String> collectionNames = new LinkedHashSet<>(); // consistent ordering
|
||||||
Set<String> collectionNames = new HashSet<>();
|
for (String collectionName : inputCollections) {
|
||||||
// validate collections
|
|
||||||
for (String collectionName : rawCollectionsList) {
|
|
||||||
if (stateProvider.getState(collectionName) == null) {
|
if (stateProvider.getState(collectionName) == null) {
|
||||||
String alias = stateProvider.getAlias(collectionName);
|
// perhaps it's an alias
|
||||||
if (alias != null) {
|
List<String> aliasedCollections = stateProvider.resolveAlias(collectionName);
|
||||||
List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
|
// one more level of alias indirection... (dubious that we should support this)
|
||||||
collectionNames.addAll(aliasList);
|
for (String aliasedCollection : aliasedCollections) {
|
||||||
continue;
|
collectionNames.addAll(stateProvider.resolveAlias(aliasedCollection));
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
|
collectionNames.add(collectionName); // it's a collection
|
||||||
}
|
}
|
||||||
|
|
||||||
collectionNames.add(collectionName);
|
|
||||||
}
|
}
|
||||||
return collectionNames;
|
return collectionNames;
|
||||||
}
|
}
|
||||||
|
@ -1199,7 +1160,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
DocCollection col = cacheEntry == null ? null : cacheEntry.cached;
|
DocCollection col = cacheEntry == null ? null : cacheEntry.cached;
|
||||||
if (col != null) {
|
if (col != null) {
|
||||||
if (expectedVersion <= col.getZNodeVersion()
|
if (expectedVersion <= col.getZNodeVersion()
|
||||||
&& !cacheEntry.shoulRetry()) return col;
|
&& !cacheEntry.shouldRetry()) return col;
|
||||||
}
|
}
|
||||||
|
|
||||||
CollectionRef ref = getCollectionRef(collection);
|
CollectionRef ref = getCollectionRef(collection);
|
||||||
|
@ -1220,7 +1181,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
col = cacheEntry == null ? null : cacheEntry.cached;
|
col = cacheEntry == null ? null : cacheEntry.cached;
|
||||||
if (col != null) {
|
if (col != null) {
|
||||||
if (expectedVersion <= col.getZNodeVersion()
|
if (expectedVersion <= col.getZNodeVersion()
|
||||||
&& !cacheEntry.shoulRetry()) return col;
|
&& !cacheEntry.shouldRetry()) return col;
|
||||||
}
|
}
|
||||||
// We are going to fetch a new version
|
// We are going to fetch a new version
|
||||||
// we MUST try to get a new version
|
// we MUST try to get a new version
|
||||||
|
@ -1466,7 +1427,7 @@ public class CloudSolrClient extends SolrClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tells {@link Builder} that created clients should send updats only to shard leaders.
|
* Tells {@link Builder} that created clients should send updates only to shard leaders.
|
||||||
*/
|
*/
|
||||||
public Builder sendUpdatesOnlyToShardLeaders() {
|
public Builder sendUpdatesOnlyToShardLeaders() {
|
||||||
shardLeadersOnly = true;
|
shardLeadersOnly = true;
|
||||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.impl;
|
||||||
import java.io.Closeable;
|
import java.io.Closeable;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.apache.solr.common.cloud.ClusterState;
|
import org.apache.solr.common.cloud.ClusterState;
|
||||||
|
@ -37,15 +38,10 @@ public interface ClusterStateProvider extends Closeable {
|
||||||
Set<String> getLiveNodes();
|
Set<String> getLiveNodes();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Given an alias, returns the collection name that this alias points to
|
* Given a collection alias, returns a list of collections it points to, or returns a singleton list of the input if
|
||||||
|
* it's not an alias.
|
||||||
*/
|
*/
|
||||||
String getAlias(String alias);
|
List<String> resolveAlias(String alias);
|
||||||
|
|
||||||
/**
|
|
||||||
* Given a name, returns the collection name if an alias by that name exists, or
|
|
||||||
* returns the name itself, if no alias exists.
|
|
||||||
*/
|
|
||||||
String getCollectionName(String name);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Obtain the current cluster state.
|
* Obtain the current cluster state.
|
||||||
|
|
|
@ -32,6 +32,7 @@ import org.apache.solr.client.solrj.SolrServerException;
|
||||||
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
|
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
|
||||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||||
|
import org.apache.solr.common.cloud.Aliases;
|
||||||
import org.apache.solr.common.cloud.ClusterState;
|
import org.apache.solr.common.cloud.ClusterState;
|
||||||
import org.apache.solr.common.cloud.ClusterState.CollectionRef;
|
import org.apache.solr.common.cloud.ClusterState.CollectionRef;
|
||||||
import org.apache.solr.common.cloud.ZkStateReader;
|
import org.apache.solr.common.cloud.ZkStateReader;
|
||||||
|
@ -47,7 +48,7 @@ public class HttpClusterStateProvider implements ClusterStateProvider {
|
||||||
private String urlScheme;
|
private String urlScheme;
|
||||||
volatile Set<String> liveNodes;
|
volatile Set<String> liveNodes;
|
||||||
long liveNodesTimestamp = 0;
|
long liveNodesTimestamp = 0;
|
||||||
volatile Map<String, String> aliases;
|
volatile Map<String, List<String>> aliases;
|
||||||
long aliasesTimestamp = 0;
|
long aliasesTimestamp = 0;
|
||||||
|
|
||||||
private int cacheTimeout = 5; // the liveNodes and aliases cache will be invalidated after 5 secs
|
private int cacheTimeout = 5; // the liveNodes and aliases cache will be invalidated after 5 secs
|
||||||
|
@ -191,12 +192,11 @@ public class HttpClusterStateProvider implements ClusterStateProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getAlias(String alias) {
|
public List<String> resolveAlias(String aliasName) {
|
||||||
Map<String, String> aliases = getAliases(false);
|
return Aliases.resolveAliasesGivenAliasMap(getAliases(false), aliasName);
|
||||||
return aliases.get(alias);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, String> getAliases(boolean forceFetch) {
|
private Map<String, List<String>> getAliases(boolean forceFetch) {
|
||||||
if (this.liveNodes == null) {
|
if (this.liveNodes == null) {
|
||||||
throw new RuntimeException("We don't know of any live_nodes to fetch the"
|
throw new RuntimeException("We don't know of any live_nodes to fetch the"
|
||||||
+ " latest aliases information from. "
|
+ " latest aliases information from. "
|
||||||
|
@ -212,10 +212,10 @@ public class HttpClusterStateProvider implements ClusterStateProvider {
|
||||||
withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)).
|
withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)).
|
||||||
withHttpClient(httpClient).build()) {
|
withHttpClient(httpClient).build()) {
|
||||||
|
|
||||||
Map<String, String> aliases = new CollectionAdminRequest.ListAliases().process(client).getAliases();
|
Map<String, List<String>> aliases = new CollectionAdminRequest.ListAliases().process(client).getAliasesAsLists();
|
||||||
this.aliases = aliases;
|
this.aliases = aliases;
|
||||||
this.aliasesTimestamp = System.nanoTime();
|
this.aliasesTimestamp = System.nanoTime();
|
||||||
return Collections.unmodifiableMap(aliases);
|
return Collections.unmodifiableMap(this.aliases);
|
||||||
} catch (SolrServerException | RemoteSolrException | IOException e) {
|
} catch (SolrServerException | RemoteSolrException | IOException e) {
|
||||||
// Situation where we're hitting an older Solr which doesn't have LISTALIASES
|
// Situation where we're hitting an older Solr which doesn't have LISTALIASES
|
||||||
if (e instanceof RemoteSolrException && ((RemoteSolrException)e).code()==400) {
|
if (e instanceof RemoteSolrException && ((RemoteSolrException)e).code()==400) {
|
||||||
|
@ -240,12 +240,6 @@ public class HttpClusterStateProvider implements ClusterStateProvider {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCollectionName(String name) {
|
|
||||||
Map<String, String> aliases = getAliases(false);
|
|
||||||
return aliases.containsKey(name) ? aliases.get(name): name;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ClusterState getClusterState() throws IOException {
|
public ClusterState getClusterState() throws IOException {
|
||||||
for (String nodeName: liveNodes) {
|
for (String nodeName: liveNodes) {
|
||||||
|
|
|
@ -22,11 +22,11 @@ import java.lang.invoke.MethodHandles;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.apache.solr.common.SolrException;
|
import org.apache.solr.common.SolrException;
|
||||||
import org.apache.solr.common.cloud.Aliases;
|
|
||||||
import org.apache.solr.common.cloud.ClusterState;
|
import org.apache.solr.common.cloud.ClusterState;
|
||||||
import org.apache.solr.common.cloud.ZkStateReader;
|
import org.apache.solr.common.cloud.ZkStateReader;
|
||||||
import org.apache.solr.common.cloud.ZooKeeperException;
|
import org.apache.solr.common.cloud.ZooKeeperException;
|
||||||
|
@ -83,9 +83,8 @@ public class ZkClientClusterStateProvider implements ClusterStateProvider {
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getAlias(String alias) {
|
public List<String> resolveAlias(String alias) {
|
||||||
Aliases aliases = zkStateReader.getAliases();
|
return zkStateReader.getAliases().resolveAliases(alias); // if not an alias, returns itself
|
||||||
return aliases.getCollectionAlias(alias);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -103,18 +102,6 @@ public class ZkClientClusterStateProvider implements ClusterStateProvider {
|
||||||
return def;
|
return def;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCollectionName(String name) {
|
|
||||||
Aliases aliases = zkStateReader.getAliases();
|
|
||||||
if (aliases != null) {
|
|
||||||
Map<String, String> collectionAliases = aliases.getCollectionAliasMap();
|
|
||||||
if (collectionAliases != null && collectionAliases.containsKey(name)) {
|
|
||||||
name = collectionAliases.get(name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ClusterState getClusterState() throws IOException {
|
public ClusterState getClusterState() throws IOException {
|
||||||
return zkStateReader.getClusterState();
|
return zkStateReader.getClusterState();
|
||||||
|
|
|
@ -26,6 +26,7 @@ import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
|
import java.util.Objects;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
|
@ -46,7 +47,6 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
|
||||||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
|
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
|
||||||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
|
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue;
|
||||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||||
import org.apache.solr.common.cloud.Aliases;
|
|
||||||
import org.apache.solr.common.cloud.ClusterState;
|
import org.apache.solr.common.cloud.ClusterState;
|
||||||
import org.apache.solr.common.cloud.DocCollection;
|
import org.apache.solr.common.cloud.DocCollection;
|
||||||
import org.apache.solr.common.cloud.Slice;
|
import org.apache.solr.common.cloud.Slice;
|
||||||
|
@ -55,7 +55,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
|
||||||
import org.apache.solr.common.params.SolrParams;
|
import org.apache.solr.common.params.SolrParams;
|
||||||
import org.apache.solr.common.util.ExecutorUtil;
|
import org.apache.solr.common.util.ExecutorUtil;
|
||||||
import org.apache.solr.common.util.SolrjNamedThreadFactory;
|
import org.apache.solr.common.util.SolrjNamedThreadFactory;
|
||||||
import org.apache.solr.common.util.StrUtils;
|
|
||||||
|
|
||||||
import static org.apache.solr.common.params.CommonParams.DISTRIB;
|
import static org.apache.solr.common.params.CommonParams.DISTRIB;
|
||||||
import static org.apache.solr.common.params.CommonParams.SORT;
|
import static org.apache.solr.common.params.CommonParams.SORT;
|
||||||
|
@ -331,9 +330,21 @@ public class CloudSolrStream extends TupleStream implements Expressible {
|
||||||
|
|
||||||
Map<String, DocCollection> collectionsMap = clusterState.getCollectionsMap();
|
Map<String, DocCollection> collectionsMap = clusterState.getCollectionsMap();
|
||||||
|
|
||||||
// Check collection case sensitive
|
//TODO we should probably split collection by comma to query more than one
|
||||||
if(collectionsMap.containsKey(collectionName)) {
|
// which is something already supported in other parts of Solr
|
||||||
return collectionsMap.get(collectionName).getActiveSlices();
|
|
||||||
|
// check for alias or collection
|
||||||
|
List<String> collections = checkAlias
|
||||||
|
? zkStateReader.getAliases().resolveAliases(collectionName) // if not an alias, returns collectionName
|
||||||
|
: Collections.singletonList(collectionName);
|
||||||
|
// Lookup all actives slices for these collections
|
||||||
|
List<Slice> slices = collections.stream()
|
||||||
|
.map(collectionsMap::get)
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.flatMap(docCol -> docCol.getActiveSlices().stream())
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
if (!slices.isEmpty()) {
|
||||||
|
return slices;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check collection case insensitive
|
// Check collection case insensitive
|
||||||
|
@ -343,23 +354,6 @@ public class CloudSolrStream extends TupleStream implements Expressible {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(checkAlias) {
|
|
||||||
// check for collection alias
|
|
||||||
Aliases aliases = zkStateReader.getAliases();
|
|
||||||
String alias = aliases.getCollectionAlias(collectionName);
|
|
||||||
if (alias != null) {
|
|
||||||
Collection<Slice> slices = new ArrayList<>();
|
|
||||||
|
|
||||||
List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
|
|
||||||
for (String aliasCollectionName : aliasList) {
|
|
||||||
// Add all active slices for this alias collection
|
|
||||||
slices.addAll(collectionsMap.get(aliasCollectionName).getActiveSlices());
|
|
||||||
}
|
|
||||||
|
|
||||||
return slices;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new IOException("Slices not found for " + collectionName);
|
throw new IOException("Slices not found for " + collectionName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -37,14 +37,11 @@ import org.apache.solr.client.solrj.io.stream.expr.Explanation;
|
||||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||||
import org.apache.solr.common.IteratorWriter;
|
import org.apache.solr.common.IteratorWriter;
|
||||||
import org.apache.solr.common.MapWriter;
|
import org.apache.solr.common.MapWriter;
|
||||||
import org.apache.solr.common.cloud.Aliases;
|
|
||||||
import org.apache.solr.common.cloud.ClusterState;
|
import org.apache.solr.common.cloud.ClusterState;
|
||||||
import org.apache.solr.common.cloud.DocCollection;
|
|
||||||
import org.apache.solr.common.cloud.Replica;
|
import org.apache.solr.common.cloud.Replica;
|
||||||
import org.apache.solr.common.cloud.Slice;
|
import org.apache.solr.common.cloud.Slice;
|
||||||
import org.apache.solr.common.cloud.ZkCoreNodeProps;
|
import org.apache.solr.common.cloud.ZkCoreNodeProps;
|
||||||
import org.apache.solr.common.cloud.ZkStateReader;
|
import org.apache.solr.common.cloud.ZkStateReader;
|
||||||
import org.apache.solr.common.util.StrUtils;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@ -141,7 +138,7 @@ public abstract class TupleStream implements Closeable, Serializable, MapWriter
|
||||||
CloudSolrClient cloudSolrClient = streamContext.getSolrClientCache().getCloudSolrClient(zkHost);
|
CloudSolrClient cloudSolrClient = streamContext.getSolrClientCache().getCloudSolrClient(zkHost);
|
||||||
ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
|
ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
|
||||||
ClusterState clusterState = zkStateReader.getClusterState();
|
ClusterState clusterState = zkStateReader.getClusterState();
|
||||||
Collection<Slice> slices = getSlices(collection, zkStateReader, true);
|
Collection<Slice> slices = CloudSolrStream.getSlices(collection, zkStateReader, true);
|
||||||
Set<String> liveNodes = clusterState.getLiveNodes();
|
Set<String> liveNodes = clusterState.getLiveNodes();
|
||||||
for(Slice slice : slices) {
|
for(Slice slice : slices) {
|
||||||
Collection<Replica> replicas = slice.getReplicas();
|
Collection<Replica> replicas = slice.getReplicas();
|
||||||
|
@ -162,45 +159,6 @@ public abstract class TupleStream implements Closeable, Serializable, MapWriter
|
||||||
return shards;
|
return shards;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Collection<Slice> getSlices(String collectionName,
|
|
||||||
ZkStateReader zkStateReader,
|
|
||||||
boolean checkAlias) throws IOException {
|
|
||||||
ClusterState clusterState = zkStateReader.getClusterState();
|
|
||||||
|
|
||||||
Map<String, DocCollection> collectionsMap = clusterState.getCollectionsMap();
|
|
||||||
|
|
||||||
// Check collection case sensitive
|
|
||||||
if(collectionsMap.containsKey(collectionName)) {
|
|
||||||
return collectionsMap.get(collectionName).getActiveSlices();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check collection case insensitive
|
|
||||||
for(String collectionMapKey : collectionsMap.keySet()) {
|
|
||||||
if(collectionMapKey.equalsIgnoreCase(collectionName)) {
|
|
||||||
return collectionsMap.get(collectionMapKey).getActiveSlices();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if(checkAlias) {
|
|
||||||
// check for collection alias
|
|
||||||
Aliases aliases = zkStateReader.getAliases();
|
|
||||||
String alias = aliases.getCollectionAlias(collectionName);
|
|
||||||
if (alias != null) {
|
|
||||||
Collection<Slice> slices = new ArrayList<>();
|
|
||||||
|
|
||||||
List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
|
|
||||||
for (String aliasCollectionName : aliasList) {
|
|
||||||
// Add all active slices for this alias collection
|
|
||||||
slices.addAll(collectionsMap.get(aliasCollectionName).getActiveSlices());
|
|
||||||
}
|
|
||||||
|
|
||||||
return slices;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new IOException("Slices not found for " + collectionName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class IgnoreException extends IOException {
|
public static class IgnoreException extends IOException {
|
||||||
public void printStackTrace(PrintWriter pw) {
|
public void printStackTrace(PrintWriter pw) {
|
||||||
pw.print("Early Client Disconnect");
|
pw.print("Early Client Disconnect");
|
||||||
|
|
|
@ -18,8 +18,10 @@ package org.apache.solr.client.solrj.response;
|
||||||
|
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.solr.common.cloud.Aliases;
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
|
|
||||||
public class CollectionAdminResponse extends SolrResponseBase
|
public class CollectionAdminResponse extends SolrResponseBase
|
||||||
|
@ -76,6 +78,11 @@ public class CollectionAdminResponse extends SolrResponseBase
|
||||||
return Collections.emptyMap();
|
return Collections.emptyMap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<String, List<String>> getAliasesAsLists() {
|
||||||
|
// TODO we compute on each call... should this be done once & cached?
|
||||||
|
return Aliases.convertMapOfCommaDelimitedToMapOfList(getAliases());
|
||||||
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public Map<String, NamedList<Integer>> getCollectionNodesStatus()
|
public Map<String, NamedList<Integer>> getCollectionNodesStatus()
|
||||||
{
|
{
|
||||||
|
|
|
@ -16,47 +16,132 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.solr.common.cloud;
|
package org.apache.solr.common.cloud;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.solr.common.util.StrUtils;
|
||||||
|
import org.apache.solr.common.util.Utils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Holds collection aliases -- virtual collections that point to one or more other collections.
|
||||||
|
* We might add other types of aliases here some day.
|
||||||
|
* Immutable.
|
||||||
|
*/
|
||||||
public class Aliases {
|
public class Aliases {
|
||||||
|
|
||||||
private Map<String,Map<String,String>> aliasMap;
|
public static final Aliases EMPTY = new Aliases(Collections.emptyMap());
|
||||||
|
|
||||||
public Aliases(Map<String,Map<String,String>> aliasMap) {
|
/** Map of "collection" string constant to ->
|
||||||
|
* alias name -> comma delimited list of collections */
|
||||||
|
private final Map<String,Map<String,String>> aliasMap; // not-null
|
||||||
|
|
||||||
|
private final Map<String, List<String>> collectionAliasListMap; // not-null; computed from aliasMap
|
||||||
|
|
||||||
|
public static Aliases fromJSON(byte[] bytes) {
|
||||||
|
if (bytes == null || bytes.length == 0) {
|
||||||
|
return EMPTY;
|
||||||
|
}
|
||||||
|
return new Aliases((Map<String,Map<String,String>>) Utils.fromJSON(bytes));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Aliases(Map<String, Map<String,String>> aliasMap) {
|
||||||
this.aliasMap = aliasMap;
|
this.aliasMap = aliasMap;
|
||||||
|
collectionAliasListMap = convertMapOfCommaDelimitedToMapOfList(getCollectionAliasMap());
|
||||||
}
|
}
|
||||||
|
|
||||||
public Aliases() {
|
public static Map<String, List<String>> convertMapOfCommaDelimitedToMapOfList(Map<String, String> collectionAliasMap) {
|
||||||
this.aliasMap = new HashMap<>();
|
Map<String, List<String>> collectionAliasListMap = new LinkedHashMap<>(collectionAliasMap.size());
|
||||||
|
for (Map.Entry<String, String> entry : collectionAliasMap.entrySet()) {
|
||||||
|
collectionAliasListMap.put(entry.getKey(), StrUtils.splitSmart(entry.getValue(), ",", true));
|
||||||
|
}
|
||||||
|
return collectionAliasListMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns an unmodifiable Map of collection aliases mapped to a comma delimited list of what the alias maps to.
|
||||||
|
* Does not return null.
|
||||||
|
* Prefer use of {@link #getCollectionAliasListMap()} instead, where appropriate.
|
||||||
|
*/
|
||||||
public Map<String,String> getCollectionAliasMap() {
|
public Map<String,String> getCollectionAliasMap() {
|
||||||
Map<String,String> cam = aliasMap.get("collection");
|
Map<String,String> cam = aliasMap.get("collection");
|
||||||
if (cam == null) return null;
|
return cam == null ? Collections.emptyMap() : Collections.unmodifiableMap(cam);
|
||||||
return Collections.unmodifiableMap(cam);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<String,Map<String,String>> getAliasMap() {
|
/**
|
||||||
return Collections.unmodifiableMap(aliasMap);
|
* Returns an unmodifiable Map of collection aliases mapped to a list of what the alias maps to.
|
||||||
|
* Does not return null.
|
||||||
|
*/
|
||||||
|
public Map<String,List<String>> getCollectionAliasListMap() {
|
||||||
|
return Collections.unmodifiableMap(collectionAliasListMap);
|
||||||
}
|
}
|
||||||
|
|
||||||
public int collectionAliasSize() {
|
public boolean hasCollectionAliases() {
|
||||||
Map<String,String> cam = aliasMap.get("collection");
|
return !collectionAliasListMap.isEmpty();
|
||||||
if (cam == null) return 0;
|
}
|
||||||
return cam.size();
|
|
||||||
|
/**
|
||||||
|
* Returns a list of collections that the input alias name maps to. If there
|
||||||
|
* are none, the input is returned. One level of alias indirection is supported (alias to alias to collection).
|
||||||
|
* Treat the result as unmodifiable.
|
||||||
|
*/
|
||||||
|
public List<String> resolveAliases(String aliasName) {
|
||||||
|
return resolveAliasesGivenAliasMap(collectionAliasListMap, aliasName);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @lucene.internal */
|
||||||
|
public static List<String> resolveAliasesGivenAliasMap(Map<String, List<String>> collectionAliasListMap, String aliasName) {
|
||||||
|
//return collectionAliasListMap.getOrDefault(aliasName, Collections.singletonList(aliasName));
|
||||||
|
// TODO deprecate and remove this dubious feature?
|
||||||
|
// Due to another level of indirection, this is more complicated...
|
||||||
|
List<String> level1 = collectionAliasListMap.get(aliasName);
|
||||||
|
if (level1 == null) {
|
||||||
|
return Collections.singletonList(aliasName);// is a collection
|
||||||
|
}
|
||||||
|
List<String> result = new ArrayList<>(level1.size());
|
||||||
|
for (String level1Alias : level1) {
|
||||||
|
List<String> level2 = collectionAliasListMap.get(level1Alias);
|
||||||
|
if (level2 == null) {
|
||||||
|
result.add(level1Alias);
|
||||||
|
} else {
|
||||||
|
result.addAll(level2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new Aliases instance with the same data as the current one but with a modification based on the
|
||||||
|
* parameters. If {@code collections} is null, then the {@code alias} is removed, otherwise it is added/updated.
|
||||||
|
*/
|
||||||
|
public Aliases cloneWithCollectionAlias(String alias, String collections) {
|
||||||
|
Map<String,String> newCollectionMap = new HashMap<>(getCollectionAliasMap());
|
||||||
|
if (collections == null) {
|
||||||
|
newCollectionMap.remove(alias);
|
||||||
|
} else {
|
||||||
|
newCollectionMap.put(alias, collections);
|
||||||
|
}
|
||||||
|
if (newCollectionMap.isEmpty()) {
|
||||||
|
return EMPTY;
|
||||||
|
} else {
|
||||||
|
return new Aliases(Collections.singletonMap("collection", newCollectionMap));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Serialize to ZooKeeper. */
|
||||||
|
public byte[] toJSON() {
|
||||||
|
if (collectionAliasListMap.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
} else {
|
||||||
|
return Utils.toJSON(aliasMap);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "Aliases [aliasMap=" + aliasMap + "]";
|
return "Aliases [aliasMap=" + aliasMap + "]";
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getCollectionAlias(String collectionName) {
|
|
||||||
Map<String,String> cam = aliasMap.get("collection");
|
|
||||||
if (cam == null) return null;
|
|
||||||
return cam.get(collectionName);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -232,15 +232,6 @@ public class ClusterState implements JSONWriter.Writable {
|
||||||
return new ClusterState( liveNodes, collections,version);
|
return new ClusterState( liveNodes, collections,version);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Aliases load(byte[] bytes) {
|
|
||||||
if (bytes == null || bytes.length == 0) {
|
|
||||||
return new Aliases();
|
|
||||||
}
|
|
||||||
Map<String,Map<String,String>> aliasMap = (Map<String,Map<String,String>>) Utils.fromJSON(bytes);
|
|
||||||
|
|
||||||
return new Aliases(aliasMap);
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO move to static DocCollection.loadFromMap
|
// TODO move to static DocCollection.loadFromMap
|
||||||
private static DocCollection collectionFromObjects(String name, Map<String, Object> objs, Integer version, String znode) {
|
private static DocCollection collectionFromObjects(String name, Map<String, Object> objs, Integer version, String znode) {
|
||||||
Map<String,Object> props;
|
Map<String,Object> props;
|
||||||
|
|
|
@ -257,7 +257,7 @@ public class ZkStateReader implements Closeable {
|
||||||
|
|
||||||
private final boolean closeClient;
|
private final boolean closeClient;
|
||||||
|
|
||||||
private volatile Aliases aliases = new Aliases();
|
private volatile Aliases aliases = Aliases.EMPTY;
|
||||||
|
|
||||||
private volatile boolean closed = false;
|
private volatile boolean closed = false;
|
||||||
|
|
||||||
|
@ -386,7 +386,9 @@ public class ZkStateReader implements Closeable {
|
||||||
refreshLiveNodes(null);
|
refreshLiveNodes(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Never null. */
|
||||||
public Aliases getAliases() {
|
public Aliases getAliases() {
|
||||||
|
assert aliases != null;
|
||||||
return aliases;
|
return aliases;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -455,7 +457,7 @@ public class ZkStateReader implements Closeable {
|
||||||
final Watcher thisWatch = this;
|
final Watcher thisWatch = this;
|
||||||
final Stat stat = new Stat();
|
final Stat stat = new Stat();
|
||||||
final byte[] data = zkClient.getData(ALIASES, thisWatch, stat, true);
|
final byte[] data = zkClient.getData(ALIASES, thisWatch, stat, true);
|
||||||
ZkStateReader.this.aliases = ClusterState.load(data);
|
ZkStateReader.this.aliases = Aliases.fromJSON(data);
|
||||||
LOG.debug("New alias definition is: " + ZkStateReader.this.aliases.toString());
|
LOG.debug("New alias definition is: " + ZkStateReader.this.aliases.toString());
|
||||||
}
|
}
|
||||||
} catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
|
} catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
|
||||||
|
@ -896,7 +898,7 @@ public class ZkStateReader implements Closeable {
|
||||||
|
|
||||||
public void updateAliases() throws KeeperException, InterruptedException {
|
public void updateAliases() throws KeeperException, InterruptedException {
|
||||||
final byte[] data = zkClient.getData(ALIASES, null, null, true);
|
final byte[] data = zkClient.getData(ALIASES, null, null, true);
|
||||||
this.aliases = ClusterState.load(data);
|
this.aliases = Aliases.fromJSON(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -88,6 +88,7 @@ public class StrUtils {
|
||||||
* @param s the string to split
|
* @param s the string to split
|
||||||
* @param separator the separator to split on
|
* @param separator the separator to split on
|
||||||
* @param decode decode backslash escaping
|
* @param decode decode backslash escaping
|
||||||
|
* @return not null
|
||||||
*/
|
*/
|
||||||
public static List<String> splitSmart(String s, String separator, boolean decode) {
|
public static List<String> splitSmart(String s, String separator, boolean decode) {
|
||||||
ArrayList<String> lst = new ArrayList<>(2);
|
ArrayList<String> lst = new ArrayList<>(2);
|
||||||
|
|
|
@ -22,6 +22,7 @@ import java.net.SocketException;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
@ -130,13 +131,8 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getAlias(String collection) {
|
public List<String> resolveAlias(String collection) {
|
||||||
return collection;
|
return Collections.singletonList(collection);
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCollectionName(String name) {
|
|
||||||
return name;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -28,7 +28,6 @@ import java.sql.Types;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.SortedSet;
|
import java.util.SortedSet;
|
||||||
|
@ -555,13 +554,7 @@ public class JdbcTest extends SolrCloudTestCase {
|
||||||
tables.addAll(collectionsSet);
|
tables.addAll(collectionsSet);
|
||||||
|
|
||||||
Aliases aliases = zkStateReader.getAliases();
|
Aliases aliases = zkStateReader.getAliases();
|
||||||
if(aliases != null) {
|
tables.addAll(aliases.getCollectionAliasListMap().keySet());
|
||||||
Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap();
|
|
||||||
if(collectionAliasMap != null) {
|
|
||||||
Set<String> aliasesSet = collectionAliasMap.keySet();
|
|
||||||
tables.addAll(aliasesSet);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try(ResultSet rs = databaseMetaData.getTables(null, zkHost, "%", null)) {
|
try(ResultSet rs = databaseMetaData.getTables(null, zkHost, "%", null)) {
|
||||||
for(String table : tables) {
|
for(String table : tables) {
|
||||||
|
|
|
@ -84,6 +84,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
|
||||||
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
|
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
|
||||||
import org.apache.solr.client.solrj.util.ClientUtils;
|
import org.apache.solr.client.solrj.util.ClientUtils;
|
||||||
import org.apache.solr.cloud.IpTables;
|
import org.apache.solr.cloud.IpTables;
|
||||||
|
import org.apache.solr.cloud.MiniSolrCloudCluster;
|
||||||
import org.apache.solr.common.SolrDocument;
|
import org.apache.solr.common.SolrDocument;
|
||||||
import org.apache.solr.common.SolrDocumentList;
|
import org.apache.solr.common.SolrDocumentList;
|
||||||
import org.apache.solr.common.SolrException;
|
import org.apache.solr.common.SolrException;
|
||||||
|
@ -2224,6 +2225,14 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
||||||
super();
|
super();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public CloudSolrClient.Builder withCluster(MiniSolrCloudCluster cluster) {
|
||||||
|
if (random().nextBoolean()) {
|
||||||
|
return withZkHost(cluster.getZkServer().getZkAddress());
|
||||||
|
} else {
|
||||||
|
return withSolrUrl(cluster.getRandomJetty(random()).getBaseUrl().toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public CloudSolrClient.Builder sendDirectUpdatesToShardLeadersOnly() {
|
public CloudSolrClient.Builder sendDirectUpdatesToShardLeadersOnly() {
|
||||||
configuredDUTflag = true;
|
configuredDUTflag = true;
|
||||||
|
@ -2247,7 +2256,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
||||||
@Override
|
@Override
|
||||||
public CloudSolrClient build() {
|
public CloudSolrClient build() {
|
||||||
if (configuredDUTflag == false) {
|
if (configuredDUTflag == false) {
|
||||||
// flag value not explicity configured
|
// flag value not explicitly configured
|
||||||
if (random().nextBoolean()) {
|
if (random().nextBoolean()) {
|
||||||
// so randomly choose a value
|
// so randomly choose a value
|
||||||
randomlyChooseDirectUpdatesToLeadersOnly();
|
randomlyChooseDirectUpdatesToLeadersOnly();
|
||||||
|
@ -2271,6 +2280,17 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
|
||||||
|
* Tests that do not wish to have any randomized behavior should use the
|
||||||
|
* {@link org.apache.solr.client.solrj.impl.CloudSolrClient.Builder} class directly
|
||||||
|
*/
|
||||||
|
public static CloudSolrClient getCloudSolrClient(MiniSolrCloudCluster cluster) {
|
||||||
|
return new CloudSolrClientBuilder()
|
||||||
|
.withCluster(cluster)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
|
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
|
||||||
* Tests that do not wish to have any randomized behavior should use the
|
* Tests that do not wish to have any randomized behavior should use the
|
||||||
|
@ -2301,6 +2321,10 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static CloudSolrClientBuilder newCloudSolrClient(String zkHost) {
|
||||||
|
return (CloudSolrClientBuilder) new CloudSolrClientBuilder().withZkHost(zkHost);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
|
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
|
||||||
* Tests that do not wish to have any randomized behavior should use the
|
* Tests that do not wish to have any randomized behavior should use the
|
||||||
|
|
Loading…
Reference in New Issue