mirror of https://github.com/apache/lucene.git
SOLR-10447, SOLR-10447: LISTALIASES Collections API command; CloudSolrClient can be initialized using Solr URL
SOLR-10447: Collections API now supports a LISTALIASES command to return a list of all collection aliases.
SOLR-10446: CloudSolrClient can now be initialized using the base URL of a Solr instance instead of
ZooKeeper hosts. This is possible through the use of newly introduced HttpClusterStateProvider.
To fetch a list of collection aliases, this depends on LISTALIASES command, and hence this way of
initializing CloudSolrClient would not work with older versions of Solr that doesn't support LISTALIASES.
This commit is contained in:
Ishan Chattopadhyaya
parent
00f0c3022b
commit
4df4c52c0c
|
|
@ -158,6 +158,15 @@ New Features
|
|||
|
||||
* SOLR-9936: Allow configuration for recoveryExecutor thread pool size. (Tim Owen via Mark Miller)
|
||||
|
||||
* SOLR-10447: Collections API now supports a LISTALIASES command to return a list of all collection aliases.
|
||||
(Ishan Chattopadhyaya)
|
||||
|
||||
* SOLR-10446: CloudSolrClient can now be initialized using the base URL of a Solr instance instead of
|
||||
ZooKeeper hosts. This is possible through the use of newly introduced HttpClusterStateProvider.
|
||||
To fetch a list of collection aliases, this depends on LISTALIASES command, and hence this way of
|
||||
initializing CloudSolrClient would not work with older versions of Solr that doesn't support LISTALIASES.
|
||||
(Ishan Chattopadhyaya, Noble Paul)
|
||||
|
||||
Optimizations
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
|
|||
GET_CLUSTER_STATUS_CMD(EndPoint.CLUSTER_CMD_STATUS, GET, REQUESTSTATUS_OP),
|
||||
DELETE_CLUSTER_STATUS(EndPoint.CLUSTER_CMD_STATUS_DELETE, DELETE, DELETESTATUS_OP),
|
||||
GET_A_COLLECTION(EndPoint.COLLECTION_STATE, GET, CLUSTERSTATUS_OP),
|
||||
LIST_ALIASES(EndPoint.CLUSTER_ALIASES, GET, LISTALIASES_OP),
|
||||
CREATE_COLLECTION(EndPoint.COLLECTIONS_COMMANDS,
|
||||
POST,
|
||||
CREATE_OP,
|
||||
|
|
@ -290,6 +291,7 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
|
|||
|
||||
enum EndPoint implements V2EndPoint {
|
||||
CLUSTER("cluster"),
|
||||
CLUSTER_ALIASES("cluster.aliases"),
|
||||
CLUSTER_CMD("cluster.Commands"),
|
||||
CLUSTER_NODES("cluster.nodes"),
|
||||
CLUSTER_CMD_STATUS("cluster.commandstatus"),
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ import org.apache.solr.cloud.rule.ReplicaAssigner;
|
|||
import org.apache.solr.cloud.rule.Rule;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.cloud.Aliases;
|
||||
import org.apache.solr.common.cloud.ClusterProperties;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
|
|
@ -460,6 +461,19 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
|
|||
return req.getParams().required().getAll(null, NAME, "collections");
|
||||
}),
|
||||
DELETEALIAS_OP(DELETEALIAS, (req, rsp, h) -> req.getParams().required().getAll(null, NAME)),
|
||||
|
||||
/**
|
||||
* Handle cluster status request.
|
||||
* Can return status per specific collection/shard or per all collections.
|
||||
*/
|
||||
LISTALIASES_OP(LISTALIASES, (req, rsp, h) -> {
|
||||
ZkStateReader zkStateReader = h.coreContainer.getZkController().getZkStateReader();
|
||||
Aliases aliases = zkStateReader.getAliases();
|
||||
if (aliases != null) {
|
||||
rsp.getValues().add("aliases", aliases.getCollectionAliasMap());
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
SPLITSHARD_OP(SPLITSHARD, DEFAULT_COLLECTION_OP_TIMEOUT * 5, true, (req, rsp, h) -> {
|
||||
String name = req.getParams().required().get(COLLECTION_PROP);
|
||||
// TODO : add support for multiple shards
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"documentation": "https://cwiki.apache.org/confluence/display/solr/Collections+API",
|
||||
"description": "Provides list of collection alises.",
|
||||
"methods": [
|
||||
"GET"
|
||||
],
|
||||
"url": {
|
||||
"paths": [
|
||||
"/cluster/aliases"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -57,6 +57,10 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
|
|||
|
||||
CollectionAdminRequest.createAlias("testalias", "collection1").process(cluster.getSolrClient());
|
||||
|
||||
// ensure that the alias has been registered
|
||||
assertEquals("collection1",
|
||||
new CollectionAdminRequest.ListAliases().process(cluster.getSolrClient()).getAliases().get("testalias"));
|
||||
|
||||
// search for alias
|
||||
QueryResponse res = cluster.getSolrClient().query("testalias", new SolrQuery("*:*"));
|
||||
assertEquals(3, res.getResults().getNumFound());
|
||||
|
|
|
|||
|
|
@ -393,7 +393,7 @@ public class CloudSolrClient extends SolrClient {
|
|||
*/
|
||||
@Deprecated
|
||||
public CloudSolrClient(Collection<String> zkHosts, String chroot, HttpClient httpClient, LBHttpSolrClient lbSolrClient, boolean updatesToLeaders) {
|
||||
this(zkHosts, chroot, httpClient, lbSolrClient, null, updatesToLeaders, false, null);
|
||||
this(zkHosts, chroot, null, httpClient, lbSolrClient, null, updatesToLeaders, false, null);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -407,9 +407,14 @@ public class CloudSolrClient extends SolrClient {
|
|||
* each host in the zookeeper ensemble. Note that with certain
|
||||
* Collection types like HashSet, the order of hosts in the final
|
||||
* connect string may not be in the same order you added them.
|
||||
* Provide only one of solrUrls or zkHosts.
|
||||
* @param chroot
|
||||
* A chroot value for zookeeper, starting with a forward slash. If no
|
||||
* chroot is required, use null.
|
||||
* @param solrUrls
|
||||
* A list of Solr URLs to configure the underlying {@link HttpClusterStateProvider}, which will
|
||||
* use of the these URLs to fetch the list of live nodes for this Solr cluster. Provide only
|
||||
* one of solrUrls or zkHosts.
|
||||
* @param httpClient
|
||||
* the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a
|
||||
* multi-threaded connection manager. If null, a default HttpClient will be used.
|
||||
|
|
@ -424,6 +429,7 @@ public class CloudSolrClient extends SolrClient {
|
|||
*/
|
||||
private CloudSolrClient(Collection<String> zkHosts,
|
||||
String chroot,
|
||||
List<String> solrUrls,
|
||||
HttpClient httpClient,
|
||||
LBHttpSolrClient lbSolrClient,
|
||||
LBHttpSolrClient.Builder lbHttpSolrClientBuilder,
|
||||
|
|
@ -433,7 +439,21 @@ public class CloudSolrClient extends SolrClient {
|
|||
|
||||
) {
|
||||
if (stateProvider == null) {
|
||||
if (zkHosts != null && solrUrls != null) {
|
||||
throw new IllegalArgumentException("Both zkHost(s) & solrUrl(s) have been specified. Only specify one.");
|
||||
}
|
||||
if (zkHosts != null) {
|
||||
this.stateProvider = new ZkClientClusterStateProvider(zkHosts, chroot);
|
||||
} else if (solrUrls != null && !solrUrls.isEmpty()) {
|
||||
try {
|
||||
this.stateProvider = new HttpClusterStateProvider(solrUrls, httpClient);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Couldn't initialize a HttpClusterStateProvider (is/are the "
|
||||
+ "Solr server(s), " + solrUrls + ", down?)", e);
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException("Both zkHosts and solrUrl cannot be null.");
|
||||
}
|
||||
} else {
|
||||
this.stateProvider = stateProvider;
|
||||
}
|
||||
|
|
@ -1259,7 +1279,7 @@ public class CloudSolrClient extends SolrClient {
|
|||
Set<String> liveNodes = stateProvider.liveNodes();
|
||||
for (String liveNode : liveNodes) {
|
||||
theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode,
|
||||
(String) stateProvider.getClusterProperties().getOrDefault(ZkStateReader.URL_SCHEME,"http")));
|
||||
(String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
|
||||
}
|
||||
} else {
|
||||
|
||||
|
|
@ -1365,7 +1385,7 @@ public class CloudSolrClient extends SolrClient {
|
|||
return rsp.getResponse();
|
||||
}
|
||||
|
||||
Set<String> getCollectionNames(String collection) {
|
||||
private Set<String> getCollectionNames(String collection) {
|
||||
// Extract each comma separated collection name and store in a List.
|
||||
List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true);
|
||||
Set<String> collectionNames = new HashSet<>();
|
||||
|
|
@ -1602,6 +1622,7 @@ public class CloudSolrClient extends SolrClient {
|
|||
*/
|
||||
public static class Builder {
|
||||
private Collection<String> zkHosts;
|
||||
private List<String> solrUrls;
|
||||
private HttpClient httpClient;
|
||||
private String zkChroot;
|
||||
private LBHttpSolrClient loadBalancedSolrClient;
|
||||
|
|
@ -1613,6 +1634,7 @@ public class CloudSolrClient extends SolrClient {
|
|||
|
||||
public Builder() {
|
||||
this.zkHosts = new ArrayList();
|
||||
this.solrUrls = new ArrayList();
|
||||
this.shardLeadersOnly = true;
|
||||
}
|
||||
|
||||
|
|
@ -1630,6 +1652,27 @@ public class CloudSolrClient extends SolrClient {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide a Solr URL to be used when configuring {@link CloudSolrClient} instances.
|
||||
*
|
||||
* Method may be called multiple times. One of the provided values will be used to fetch
|
||||
* the list of live Solr nodes that the underlying {@link HttpClusterStateProvider} would be maintaining.
|
||||
*/
|
||||
public Builder withSolrUrl(String solrUrl) {
|
||||
this.solrUrls.add(solrUrl);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide a list of Solr URL to be used when configuring {@link CloudSolrClient} instances.
|
||||
* One of the provided values will be used to fetch the list of live Solr
|
||||
* nodes that the underlying {@link HttpClusterStateProvider} would be maintaining.
|
||||
*/
|
||||
public Builder withSolrUrl(Collection<String> solrUrls) {
|
||||
this.solrUrls.addAll(solrUrls);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a {@link HttpClient} for the builder to use when creating clients.
|
||||
*/
|
||||
|
|
@ -1722,24 +1765,51 @@ public class CloudSolrClient extends SolrClient {
|
|||
*/
|
||||
public CloudSolrClient build() {
|
||||
if (stateProvider == null) {
|
||||
if (!zkHosts.isEmpty()) {
|
||||
stateProvider = new ZkClientClusterStateProvider(zkHosts, zkChroot);
|
||||
}
|
||||
return new CloudSolrClient(zkHosts, zkChroot, httpClient, loadBalancedSolrClient, lbClientBuilder,
|
||||
else if (!this.solrUrls.isEmpty()) {
|
||||
try {
|
||||
stateProvider = new HttpClusterStateProvider(solrUrls, httpClient);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Couldn't initialize a HttpClusterStateProvider (is/are the "
|
||||
+ "Solr server(s), " + solrUrls + ", down?)", e);
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException("Both zkHosts and solrUrl cannot be null.");
|
||||
}
|
||||
}
|
||||
return new CloudSolrClient(zkHosts, zkChroot, solrUrls, httpClient, loadBalancedSolrClient, lbClientBuilder,
|
||||
shardLeadersOnly, directUpdatesToLeadersOnly, stateProvider);
|
||||
}
|
||||
}
|
||||
|
||||
interface ClusterStateProvider extends Closeable {
|
||||
|
||||
/**
|
||||
* Obtain the state of the collection (cluster status).
|
||||
* @return the collection state, or null is collection doesn't exist
|
||||
*/
|
||||
ClusterState.CollectionRef getState(String collection);
|
||||
|
||||
/**
|
||||
* Obtain set of live_nodes for the cluster.
|
||||
*/
|
||||
Set<String> liveNodes();
|
||||
|
||||
String getAlias(String collection);
|
||||
|
||||
String getCollectionName(String name);
|
||||
|
||||
Map<String, Object> getClusterProperties();
|
||||
/**
|
||||
* Obtain a cluster property, or null if it doesn't exist.
|
||||
*/
|
||||
Object getClusterProperty(String propertyName);
|
||||
|
||||
/**
|
||||
* Obtain a cluster property, or the default value if it doesn't exist.
|
||||
*/
|
||||
Object getClusterProperty(String propertyName, String def);
|
||||
|
||||
void connect();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,252 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.client.solrj.impl;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
|
||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.ClusterState.CollectionRef;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class HttpClusterStateProvider implements CloudSolrClient.ClusterStateProvider {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
private String urlScheme;
|
||||
volatile Set<String> liveNodes;
|
||||
long liveNodesTimestamp = 0;
|
||||
volatile Map<String, String> aliases;
|
||||
long aliasesTimestamp = 0;
|
||||
|
||||
private int cacheTimeout = 5; // the liveNodes and aliases cache will be invalidated after 5 secs
|
||||
final HttpClient httpClient;
|
||||
final boolean clientIsInternal;
|
||||
|
||||
public HttpClusterStateProvider(List<String> solrUrls, HttpClient httpClient) throws Exception {
|
||||
this.httpClient = httpClient == null? HttpClientUtil.createClient(null): httpClient;
|
||||
this.clientIsInternal = httpClient == null;
|
||||
for (String solrUrl: solrUrls) {
|
||||
urlScheme = solrUrl.startsWith("https")? "https": "http";
|
||||
try (SolrClient initialClient = new HttpSolrClient.Builder().withBaseSolrUrl(solrUrl).withHttpClient(httpClient).build()) {
|
||||
Set<String> liveNodes = fetchLiveNodes(initialClient); // throws exception if unable to fetch
|
||||
this.liveNodes = liveNodes;
|
||||
liveNodesTimestamp = System.nanoTime();
|
||||
break;
|
||||
} catch (IOException e) {
|
||||
log.warn("Attempt to fetch live_nodes from " + solrUrl + " failed.", e);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.liveNodes == null || this.liveNodes.isEmpty()) {
|
||||
throw new RuntimeException("Tried fetching live_nodes using Solr URLs provided, i.e. " + solrUrls + ". However, "
|
||||
+ "succeeded in obtaining the cluster state from none of them."
|
||||
+ "If you think your Solr cluster is up and is accessible,"
|
||||
+ " you could try re-creating a new CloudSolrClient using a working"
|
||||
+ " solrUrl or zkUrl.");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (this.clientIsInternal && this.httpClient != null) {
|
||||
HttpClientUtil.close(httpClient);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public CollectionRef getState(String collection) {
|
||||
for (String nodeName: liveNodes) {
|
||||
try (HttpSolrClient client = new HttpSolrClient.Builder().
|
||||
withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)).
|
||||
withHttpClient(httpClient).build()) {
|
||||
ClusterState cs = fetchClusterState(client, collection);
|
||||
return cs.getCollectionRef(collection);
|
||||
} catch (SolrServerException | RemoteSolrException | IOException e) {
|
||||
if (e.getMessage().contains(collection + " not found")) {
|
||||
// Cluster state for the given collection was not found.
|
||||
// Lets fetch/update our aliases:
|
||||
getAliases(true);
|
||||
return null;
|
||||
}
|
||||
log.warn("Attempt to fetch cluster state from " +
|
||||
ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme) + " failed.", e);
|
||||
}
|
||||
}
|
||||
throw new RuntimeException("Tried fetching cluster state using the node names we knew of, i.e. " + liveNodes +". However, "
|
||||
+ "succeeded in obtaining the cluster state from none of them."
|
||||
+ "If you think your Solr cluster is up and is accessible,"
|
||||
+ " you could try re-creating a new CloudSolrClient using a working"
|
||||
+ " solrUrl or zkUrl.");
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
private ClusterState fetchClusterState(SolrClient client, String collection) throws SolrServerException, IOException {
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.set("collection", collection);
|
||||
params.set("action", "CLUSTERSTATUS");
|
||||
QueryRequest request = new QueryRequest(params);
|
||||
request.setPath("/admin/collections");
|
||||
NamedList cluster = (SimpleOrderedMap) client.request(request).get("cluster");
|
||||
Map<String, Object> collectionsMap = ((NamedList) cluster.get("collections")).asShallowMap();
|
||||
int znodeVersion = (int)((Map<String, Object>)(collectionsMap).get(collection)).get("znodeVersion");
|
||||
Set<String> liveNodes = new HashSet((List<String>)(cluster.get("live_nodes")));
|
||||
this.liveNodes = liveNodes;
|
||||
liveNodesTimestamp = System.nanoTime();
|
||||
ClusterState cs = ClusterState.load(znodeVersion, collectionsMap, liveNodes, ZkStateReader.CLUSTER_STATE);
|
||||
return cs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> liveNodes() {
|
||||
if (liveNodes == null) {
|
||||
throw new RuntimeException("We don't know of any live_nodes to fetch the"
|
||||
+ " latest live_nodes information from. "
|
||||
+ "If you think your Solr cluster is up and is accessible,"
|
||||
+ " you could try re-creating a new CloudSolrClient using a working"
|
||||
+ " solrUrl or zkUrl.");
|
||||
}
|
||||
if (TimeUnit.SECONDS.convert((System.nanoTime() - liveNodesTimestamp), TimeUnit.NANOSECONDS) > getCacheTimeout()) {
|
||||
for (String nodeName: liveNodes) {
|
||||
try (HttpSolrClient client = new HttpSolrClient.Builder().
|
||||
withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)).
|
||||
withHttpClient(httpClient).build()) {
|
||||
Set<String> liveNodes = fetchLiveNodes(client);
|
||||
this.liveNodes = (liveNodes);
|
||||
liveNodesTimestamp = System.nanoTime();
|
||||
return liveNodes;
|
||||
} catch (Exception e) {
|
||||
log.warn("Attempt to fetch live_nodes from " +
|
||||
ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme) + " failed.", e);
|
||||
}
|
||||
}
|
||||
throw new RuntimeException("Tried fetching live_nodes using all the node names we knew of, i.e. " + liveNodes +". However, "
|
||||
+ "succeeded in obtaining the cluster state from none of them."
|
||||
+ "If you think your Solr cluster is up and is accessible,"
|
||||
+ " you could try re-creating a new CloudSolrClient using a working"
|
||||
+ " solrUrl or zkUrl.");
|
||||
} else {
|
||||
return liveNodes; // cached copy is fresh enough
|
||||
}
|
||||
}
|
||||
|
||||
private static Set<String> fetchLiveNodes(SolrClient client) throws Exception {
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.set("action", "CLUSTERSTATUS");
|
||||
QueryRequest request = new QueryRequest(params);
|
||||
request.setPath("/admin/collections");
|
||||
NamedList cluster = (SimpleOrderedMap) client.request(request).get("cluster");
|
||||
Set<String> liveNodes = new HashSet((List<String>)(cluster.get("live_nodes")));
|
||||
return liveNodes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getAlias(String collection) {
|
||||
Map<String, String> aliases = getAliases(false);
|
||||
return aliases.get(collection);
|
||||
}
|
||||
|
||||
private Map<String, String> getAliases(boolean forceFetch) {
|
||||
if (this.liveNodes == null) {
|
||||
throw new RuntimeException("We don't know of any live_nodes to fetch the"
|
||||
+ " latest aliases information from. "
|
||||
+ "If you think your Solr cluster is up and is accessible,"
|
||||
+ " you could try re-creating a new CloudSolrClient using a working"
|
||||
+ " solrUrl or zkUrl.");
|
||||
}
|
||||
|
||||
if (forceFetch || this.aliases == null ||
|
||||
TimeUnit.SECONDS.convert((System.nanoTime() - aliasesTimestamp), TimeUnit.NANOSECONDS) > getCacheTimeout()) {
|
||||
for (String nodeName: liveNodes) {
|
||||
try (HttpSolrClient client = new HttpSolrClient.Builder().
|
||||
withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)).
|
||||
withHttpClient(httpClient).build()) {
|
||||
|
||||
Map<String, String> aliases = new CollectionAdminRequest.ListAliases().process(client).getAliases();
|
||||
this.aliases = aliases;
|
||||
this.aliasesTimestamp = System.nanoTime();
|
||||
return Collections.unmodifiableMap(aliases);
|
||||
} catch (SolrServerException | RemoteSolrException | IOException e) {
|
||||
log.warn("Attempt to fetch cluster state from " +
|
||||
ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme) + " failed.", e);
|
||||
}
|
||||
}
|
||||
|
||||
throw new RuntimeException("Tried fetching aliases using all the node names we knew of, i.e. " + liveNodes +". However, "
|
||||
+ "succeeded in obtaining the cluster state from none of them."
|
||||
+ "If you think your Solr cluster is up and is accessible,"
|
||||
+ " you could try re-creating a new CloudSolrClient using a working"
|
||||
+ " solrUrl or zkUrl.");
|
||||
} else {
|
||||
return Collections.unmodifiableMap(this.aliases); // cached copy is fresh enough
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCollectionName(String name) {
|
||||
Map<String, String> aliases = getAliases(false);
|
||||
return aliases.containsKey(name) ? aliases.get(name): name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getClusterProperty(String propertyName) {
|
||||
if (propertyName.equals(ZkStateReader.URL_SCHEME)) {
|
||||
return this.urlScheme;
|
||||
}
|
||||
throw new UnsupportedOperationException("Fetching cluster properties not supported"
|
||||
+ " using the HttpClusterStateProvider. "
|
||||
+ "ZkClientClusterStateProvider can be used for this."); // TODO
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getClusterProperty(String propertyName, String def) {
|
||||
if (propertyName.equals(ZkStateReader.URL_SCHEME)) {
|
||||
return this.urlScheme;
|
||||
}
|
||||
throw new UnsupportedOperationException("Fetching cluster properties not supported"
|
||||
+ " using the HttpClusterStateProvider. "
|
||||
+ "ZkClientClusterStateProvider can be used for this."); // TODO
|
||||
}
|
||||
|
||||
@Override
|
||||
public void connect() {}
|
||||
|
||||
public int getCacheTimeout() {
|
||||
return cacheTimeout;
|
||||
}
|
||||
|
||||
public void setCacheTimeout(int cacheTimeout) {
|
||||
this.cacheTimeout = cacheTimeout;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -69,8 +69,18 @@ public class ZkClientClusterStateProvider implements CloudSolrClient.ClusterStat
|
|||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> getClusterProperties() {
|
||||
return zkStateReader.getClusterProperties();
|
||||
public Object getClusterProperty(String propertyName) {
|
||||
Map<String, Object> props = zkStateReader.getClusterProperties();
|
||||
return props.get(propertyName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getClusterProperty(String propertyName, String def) {
|
||||
Map<String, Object> props = zkStateReader.getClusterProperties();
|
||||
if (props.containsKey(propertyName)) {
|
||||
return props.get(propertyName);
|
||||
}
|
||||
return def;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -2246,6 +2246,20 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
|
|||
|
||||
}
|
||||
|
||||
// LISTALIASES request
|
||||
public static class ListAliases extends CollectionAdminRequest<CollectionAdminResponse> {
|
||||
|
||||
public ListAliases() {
|
||||
super(CollectionAction.LISTALIASES);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectionAdminResponse createResponse(SolrClient client) {
|
||||
return new CollectionAdminResponse();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a SolrRequest to get a list of collections in the cluster
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
*/
|
||||
package org.apache.solr.client.solrj.response;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -60,6 +61,16 @@ public class CollectionAdminResponse extends SolrResponseBase
|
|||
return res;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public Map<String, String> getAliases()
|
||||
{
|
||||
NamedList<Object> response = getResponse();
|
||||
if (response.get("aliases") != null) {
|
||||
return ((Map<String, String>)response.get("aliases"));
|
||||
}
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public Map<String, NamedList<Integer>> getCollectionNodesStatus()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -322,6 +322,10 @@ public class ClusterState implements JSONWriter.Writable {
|
|||
return new ClusterState(version, liveNodes, Collections.<String, DocCollection>emptyMap());
|
||||
}
|
||||
Map<String, Object> stateMap = (Map<String, Object>) Utils.fromJSON(bytes);
|
||||
return load(version, stateMap, liveNodes, znode);
|
||||
}
|
||||
|
||||
public static ClusterState load(Integer version, Map<String, Object> stateMap, Set<String> liveNodes, String znode) {
|
||||
Map<String,CollectionRef> collections = new LinkedHashMap<>(stateMap.size());
|
||||
for (Entry<String, Object> entry : stateMap.entrySet()) {
|
||||
String collectionName = entry.getKey();
|
||||
|
|
@ -332,7 +336,6 @@ public class ClusterState implements JSONWriter.Writable {
|
|||
return new ClusterState( liveNodes, collections,version);
|
||||
}
|
||||
|
||||
|
||||
public static Aliases load(byte[] bytes) {
|
||||
if (bytes == null || bytes.length == 0) {
|
||||
return new Aliases();
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ public interface CollectionParams {
|
|||
SYNCSHARD(true, LockLevel.SHARD),
|
||||
CREATEALIAS(true, LockLevel.COLLECTION),
|
||||
DELETEALIAS(true, LockLevel.COLLECTION),
|
||||
LISTALIASES(false, LockLevel.NONE),
|
||||
SPLITSHARD(true, LockLevel.SHARD),
|
||||
DELETESHARD(true, LockLevel.SHARD),
|
||||
CREATESHARD(true, LockLevel.COLLECTION),
|
||||
|
|
|
|||
|
|
@ -130,11 +130,6 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 {
|
|||
return livenodes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> getClusterProperties() {
|
||||
return Collections.EMPTY_MAP;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getAlias(String collection) {
|
||||
return collection;
|
||||
|
|
@ -152,6 +147,16 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 {
|
|||
public void close() throws IOException {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getClusterProperty(String propertyName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object getClusterProperty(String propertyName, String def) {
|
||||
return def;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ import org.apache.solr.common.util.SimpleOrderedMap;
|
|||
import org.apache.solr.handler.admin.CollectionsHandler;
|
||||
import org.apache.solr.handler.admin.ConfigSetsHandler;
|
||||
import org.apache.solr.handler.admin.CoreAdminHandler;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Rule;
|
||||
|
|
@ -90,6 +91,8 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
private static final int TIMEOUT = 30;
|
||||
private static final int NODE_COUNT = 3;
|
||||
|
||||
private static CloudSolrClient httpBasedCloudSolrClient = null;
|
||||
|
||||
@BeforeClass
|
||||
public static void setupCluster() throws Exception {
|
||||
configureCluster(NODE_COUNT)
|
||||
|
|
@ -99,8 +102,21 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1).process(cluster.getSolrClient());
|
||||
AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
|
||||
false, true, TIMEOUT);
|
||||
|
||||
httpBasedCloudSolrClient = new CloudSolrClient.Builder().withSolrUrl(
|
||||
cluster.getJettySolrRunner(0).getBaseUrl().toString()).build();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClass() {
|
||||
if (httpBasedCloudSolrClient != null) {
|
||||
try {
|
||||
httpBasedCloudSolrClient.close();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
@Before
|
||||
|
|
@ -110,6 +126,13 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
.commit(cluster.getSolrClient(), COLLECTION);
|
||||
}
|
||||
|
||||
/**
|
||||
* Randomly return the cluster's ZK based CSC, or HttpClusterProvider based CSC.
|
||||
*/
|
||||
private CloudSolrClient getRandomClient() {
|
||||
return random().nextBoolean()? cluster.getSolrClient(): httpBasedCloudSolrClient;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testParallelUpdateQTime() throws Exception {
|
||||
UpdateRequest req = new UpdateRequest();
|
||||
|
|
@ -118,7 +141,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
doc.addField("id", String.valueOf(TestUtil.nextInt(random(), 1000, 1100)));
|
||||
req.add(doc);
|
||||
}
|
||||
UpdateResponse response = req.process(cluster.getSolrClient(), COLLECTION);
|
||||
UpdateResponse response = req.process(getRandomClient(), COLLECTION);
|
||||
// See SOLR-6547, we just need to ensure that no exception is thrown here
|
||||
assertTrue(response.getQTime() >= 0);
|
||||
}
|
||||
|
|
@ -143,15 +166,31 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
.add(new SolrInputDocument(id, "1", "a_t", "hello2"), false)
|
||||
.commit(cluster.getSolrClient(), "overwrite");
|
||||
|
||||
resp = cluster.getSolrClient().query("overwrite", new SolrQuery("*:*"));
|
||||
resp = getRandomClient().query("overwrite", new SolrQuery("*:*"));
|
||||
assertEquals("There should be 3 documents because there should be two id=1 docs due to overwrite=false", 3, resp.getResults().getNumFound());
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAliasHandling() throws Exception {
|
||||
CloudSolrClient client = getRandomClient();
|
||||
SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc");
|
||||
client.add(COLLECTION, doc);
|
||||
client.commit(COLLECTION);
|
||||
|
||||
CollectionAdminRequest.createAlias("testalias", COLLECTION).process(cluster.getSolrClient());
|
||||
|
||||
// ensure that the alias has been registered
|
||||
assertEquals(COLLECTION,
|
||||
new CollectionAdminRequest.ListAliases().process(cluster.getSolrClient()).getAliases().get("testalias"));
|
||||
|
||||
assertEquals(1, client.query(COLLECTION, params("q", "*:*")).getResults().getNumFound());
|
||||
assertEquals(1, client.query("testalias", params("q", "*:*")).getResults().getNumFound());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHandlingOfStaleAlias() throws Exception {
|
||||
try (CloudSolrClient client = getCloudSolrClient(cluster.getZkServer().getZkAddress())) {
|
||||
client.setDefaultCollection("misconfigured-alias");
|
||||
CloudSolrClient client = getRandomClient();
|
||||
|
||||
CollectionAdminRequest.createCollection("nemesis", "conf", 2, 1).process(client);
|
||||
CollectionAdminRequest.createAlias("misconfigured-alias", "nemesis").process(client);
|
||||
|
|
@ -164,14 +203,13 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
docs.add(doc);
|
||||
|
||||
try {
|
||||
client.add(docs);
|
||||
client.add("misconfigured-alias", docs);
|
||||
fail("Alias points to non-existing collection, add should fail");
|
||||
} catch (SolrException e) {
|
||||
assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
|
||||
assertTrue("Unexpected exception", e.getMessage().contains("Collection not found"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRouting() throws Exception {
|
||||
|
|
@ -182,8 +220,8 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
|
||||
|
||||
// Test single threaded routed updates for UpdateRequest
|
||||
NamedList<Object> response = cluster.getSolrClient().request(request, COLLECTION);
|
||||
if (cluster.getSolrClient().isDirectUpdatesToLeadersOnly()) {
|
||||
NamedList<Object> response = getRandomClient().request(request, COLLECTION);
|
||||
if (getRandomClient().isDirectUpdatesToLeadersOnly()) {
|
||||
checkSingleServer(response);
|
||||
}
|
||||
CloudSolrClient.RouteResponse rr = (CloudSolrClient.RouteResponse) response;
|
||||
|
|
@ -214,11 +252,11 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
.deleteById("0")
|
||||
.deleteById("2")
|
||||
.commit(cluster.getSolrClient(), COLLECTION);
|
||||
if (cluster.getSolrClient().isDirectUpdatesToLeadersOnly()) {
|
||||
if (getRandomClient().isDirectUpdatesToLeadersOnly()) {
|
||||
checkSingleServer(uResponse.getResponse());
|
||||
}
|
||||
|
||||
QueryResponse qResponse = cluster.getSolrClient().query(COLLECTION, new SolrQuery("*:*"));
|
||||
QueryResponse qResponse = getRandomClient().query(COLLECTION, new SolrQuery("*:*"));
|
||||
SolrDocumentList docs = qResponse.getResults();
|
||||
assertEquals(0, docs.getNumFound());
|
||||
|
||||
|
|
@ -307,7 +345,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
ModifiableSolrParams solrParams = new ModifiableSolrParams();
|
||||
solrParams.set(CommonParams.Q, "*:*");
|
||||
solrParams.set(ShardParams._ROUTE_, sameShardRoutes.get(random().nextInt(sameShardRoutes.size())));
|
||||
log.info("output: {}", cluster.getSolrClient().query(COLLECTION, solrParams));
|
||||
log.info("output: {}", getRandomClient().query(COLLECTION, solrParams));
|
||||
}
|
||||
|
||||
// Request counts increase from expected nodes should aggregate to 1000, while there should be
|
||||
|
|
@ -362,10 +400,10 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
.add(id, "0", "a_t", "hello1")
|
||||
.add(id, "2", "a_t", "hello2")
|
||||
.add(id, "3", "a_t", "hello2")
|
||||
.commit(cluster.getSolrClient(), collectionName);
|
||||
.commit(getRandomClient(), collectionName);
|
||||
|
||||
// Run the actual test for 'preferLocalShards'
|
||||
queryWithPreferLocalShards(cluster.getSolrClient(), true, collectionName);
|
||||
queryWithPreferLocalShards(getRandomClient(), true, collectionName);
|
||||
}
|
||||
|
||||
private void queryWithPreferLocalShards(CloudSolrClient cloudClient,
|
||||
|
|
@ -658,7 +696,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
.add("id", "2", "a_t", "hello2");
|
||||
updateRequest.setParam(UpdateParams.VERSIONS, Boolean.TRUE.toString());
|
||||
|
||||
NamedList<Object> response = updateRequest.commit(cluster.getSolrClient(), COLLECTION).getResponse();
|
||||
NamedList<Object> response = updateRequest.commit(getRandomClient(), COLLECTION).getResponse();
|
||||
Object addsObject = response.get("adds");
|
||||
|
||||
assertNotNull("There must be a adds parameter", addsObject);
|
||||
|
|
@ -677,7 +715,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
assertTrue("Version for id 2 must be a long", object instanceof Long);
|
||||
versions.put("2", (Long) object);
|
||||
|
||||
QueryResponse resp = cluster.getSolrClient().query(COLLECTION, new SolrQuery("*:*"));
|
||||
QueryResponse resp = getRandomClient().query(COLLECTION, new SolrQuery("*:*"));
|
||||
assertEquals("There should be one document because overwrite=true", 2, resp.getResults().getNumFound());
|
||||
|
||||
for (SolrDocument doc : resp.getResults()) {
|
||||
|
|
@ -688,13 +726,38 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
|
|||
// assert that "deletes" are returned
|
||||
UpdateRequest deleteRequest = new UpdateRequest().deleteById("1");
|
||||
deleteRequest.setParam(UpdateParams.VERSIONS, Boolean.TRUE.toString());
|
||||
response = deleteRequest.commit(cluster.getSolrClient(), COLLECTION).getResponse();
|
||||
response = deleteRequest.commit(getRandomClient(), COLLECTION).getResponse();
|
||||
Object deletesObject = response.get("deletes");
|
||||
assertNotNull("There must be a deletes parameter", deletesObject);
|
||||
NamedList deletes = (NamedList) deletesObject;
|
||||
assertEquals("There must be 1 version", 1, deletes.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInitializationWithSolrUrls() throws Exception {
|
||||
CloudSolrClient client = httpBasedCloudSolrClient;
|
||||
SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc");
|
||||
client.add(COLLECTION, doc);
|
||||
client.commit(COLLECTION);
|
||||
assertEquals(1, client.query(COLLECTION, params("q", "*:*")).getResults().getNumFound());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCollectionDoesntExist() throws Exception {
|
||||
CloudSolrClient client = getRandomClient();
|
||||
SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc");
|
||||
try {
|
||||
client.add("boguscollectionname", doc);
|
||||
fail();
|
||||
} catch (SolrException ex) {
|
||||
if (ex.getMessage().equals("Collection not found: boguscollectionname")) {
|
||||
// pass
|
||||
} else {
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void checkSingleServer(NamedList<Object> response) {
|
||||
final CloudSolrClient.RouteResponse rr = (CloudSolrClient.RouteResponse) response;
|
||||
final Map<String,LBHttpSolrClient.Req> routes = rr.getRoutes();
|
||||
|
|
|
|||
Loading…
Reference in New Issue