From 4df4c52c0cfb8b47a066a0495bd164f6a4c973de Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Mon, 17 Apr 2017 10:11:18 +0530 Subject: [PATCH] SOLR-10447, SOLR-10447: LISTALIASES Collections API command; CloudSolrClient can be initialized using Solr URL SOLR-10447: Collections API now supports a LISTALIASES command to return a list of all collection aliases. SOLR-10446: CloudSolrClient can now be initialized using the base URL of a Solr instance instead of ZooKeeper hosts. This is possible through the use of newly introduced HttpClusterStateProvider. To fetch a list of collection aliases, this depends on LISTALIASES command, and hence this way of initializing CloudSolrClient would not work with older versions of Solr that doesn't support LISTALIASES. --- solr/CHANGES.txt | 9 + .../handler/admin/CollectionHandlerApi.java | 2 + .../handler/admin/CollectionsHandler.java | 14 + .../resources/apispec/cluster.aliases.json | 12 + .../solr/cloud/AliasIntegrationTest.java | 4 + .../client/solrj/impl/CloudSolrClient.java | 84 +++++- .../solrj/impl/HttpClusterStateProvider.java | 252 ++++++++++++++++++ .../impl/ZkClientClusterStateProvider.java | 14 +- .../solrj/request/CollectionAdminRequest.java | 14 + .../response/CollectionAdminResponse.java | 11 + .../solr/common/cloud/ClusterState.java | 5 +- .../solr/common/params/CollectionParams.java | 1 + .../solrj/impl/CloudSolrClientCacheTest.java | 15 +- .../solrj/impl/CloudSolrClientTest.java | 119 +++++++-- 14 files changed, 513 insertions(+), 43 deletions(-) create mode 100644 solr/core/src/resources/apispec/cluster.aliases.json create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index c06eaa44fe7..7d0933d17e9 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -158,6 +158,15 @@ New Features * SOLR-9936: Allow configuration for recoveryExecutor thread pool size. (Tim Owen via Mark Miller) +* SOLR-10447: Collections API now supports a LISTALIASES command to return a list of all collection aliases. + (Ishan Chattopadhyaya) + +* SOLR-10446: CloudSolrClient can now be initialized using the base URL of a Solr instance instead of + ZooKeeper hosts. This is possible through the use of newly introduced HttpClusterStateProvider. + To fetch a list of collection aliases, this depends on LISTALIASES command, and hence this way of + initializing CloudSolrClient would not work with older versions of Solr that doesn't support LISTALIASES. + (Ishan Chattopadhyaya, Noble Paul) + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java index 581fe46ba66..3cb21ab18e1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java @@ -65,6 +65,7 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport { GET_CLUSTER_STATUS_CMD(EndPoint.CLUSTER_CMD_STATUS, GET, REQUESTSTATUS_OP), DELETE_CLUSTER_STATUS(EndPoint.CLUSTER_CMD_STATUS_DELETE, DELETE, DELETESTATUS_OP), GET_A_COLLECTION(EndPoint.COLLECTION_STATE, GET, CLUSTERSTATUS_OP), + LIST_ALIASES(EndPoint.CLUSTER_ALIASES, GET, LISTALIASES_OP), CREATE_COLLECTION(EndPoint.COLLECTIONS_COMMANDS, POST, CREATE_OP, @@ -290,6 +291,7 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport { enum EndPoint implements V2EndPoint { CLUSTER("cluster"), + CLUSTER_ALIASES("cluster.aliases"), CLUSTER_CMD("cluster.Commands"), CLUSTER_NODES("cluster.nodes"), CLUSTER_CMD_STATUS("cluster.commandstatus"), diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index bb061902a66..d5c49274d74 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -52,6 +52,7 @@ import org.apache.solr.cloud.rule.ReplicaAssigner; import org.apache.solr.cloud.rule.Rule; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; @@ -460,6 +461,19 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission return req.getParams().required().getAll(null, NAME, "collections"); }), DELETEALIAS_OP(DELETEALIAS, (req, rsp, h) -> req.getParams().required().getAll(null, NAME)), + + /** + * Handle cluster status request. + * Can return status per specific collection/shard or per all collections. + */ + LISTALIASES_OP(LISTALIASES, (req, rsp, h) -> { + ZkStateReader zkStateReader = h.coreContainer.getZkController().getZkStateReader(); + Aliases aliases = zkStateReader.getAliases(); + if (aliases != null) { + rsp.getValues().add("aliases", aliases.getCollectionAliasMap()); + } + return null; + }), SPLITSHARD_OP(SPLITSHARD, DEFAULT_COLLECTION_OP_TIMEOUT * 5, true, (req, rsp, h) -> { String name = req.getParams().required().get(COLLECTION_PROP); // TODO : add support for multiple shards diff --git a/solr/core/src/resources/apispec/cluster.aliases.json b/solr/core/src/resources/apispec/cluster.aliases.json new file mode 100644 index 00000000000..9cffb714237 --- /dev/null +++ b/solr/core/src/resources/apispec/cluster.aliases.json @@ -0,0 +1,12 @@ +{ + "documentation": "https://cwiki.apache.org/confluence/display/solr/Collections+API", + "description": "Provides list of collection alises.", + "methods": [ + "GET" + ], + "url": { + "paths": [ + "/cluster/aliases" + ] + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java index 6ca072b1ba8..869650df121 100644 --- a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java @@ -57,6 +57,10 @@ public class AliasIntegrationTest extends SolrCloudTestCase { CollectionAdminRequest.createAlias("testalias", "collection1").process(cluster.getSolrClient()); + // ensure that the alias has been registered + assertEquals("collection1", + new CollectionAdminRequest.ListAliases().process(cluster.getSolrClient()).getAliases().get("testalias")); + // search for alias QueryResponse res = cluster.getSolrClient().query("testalias", new SolrQuery("*:*")); assertEquals(3, res.getResults().getNumFound()); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 4c6dd51a781..183896f2c5b 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -393,7 +393,7 @@ public class CloudSolrClient extends SolrClient { */ @Deprecated public CloudSolrClient(Collection zkHosts, String chroot, HttpClient httpClient, LBHttpSolrClient lbSolrClient, boolean updatesToLeaders) { - this(zkHosts, chroot, httpClient, lbSolrClient, null, updatesToLeaders, false, null); + this(zkHosts, chroot, null, httpClient, lbSolrClient, null, updatesToLeaders, false, null); } /** @@ -407,9 +407,14 @@ public class CloudSolrClient extends SolrClient { * each host in the zookeeper ensemble. Note that with certain * Collection types like HashSet, the order of hosts in the final * connect string may not be in the same order you added them. + * Provide only one of solrUrls or zkHosts. * @param chroot * A chroot value for zookeeper, starting with a forward slash. If no * chroot is required, use null. + * @param solrUrls + * A list of Solr URLs to configure the underlying {@link HttpClusterStateProvider}, which will + * use of the these URLs to fetch the list of live nodes for this Solr cluster. Provide only + * one of solrUrls or zkHosts. * @param httpClient * the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a * multi-threaded connection manager. If null, a default HttpClient will be used. @@ -424,6 +429,7 @@ public class CloudSolrClient extends SolrClient { */ private CloudSolrClient(Collection zkHosts, String chroot, + List solrUrls, HttpClient httpClient, LBHttpSolrClient lbSolrClient, LBHttpSolrClient.Builder lbHttpSolrClientBuilder, @@ -433,7 +439,21 @@ public class CloudSolrClient extends SolrClient { ) { if (stateProvider == null) { - this.stateProvider = new ZkClientClusterStateProvider(zkHosts, chroot); + if (zkHosts != null && solrUrls != null) { + throw new IllegalArgumentException("Both zkHost(s) & solrUrl(s) have been specified. Only specify one."); + } + if (zkHosts != null) { + this.stateProvider = new ZkClientClusterStateProvider(zkHosts, chroot); + } else if (solrUrls != null && !solrUrls.isEmpty()) { + try { + this.stateProvider = new HttpClusterStateProvider(solrUrls, httpClient); + } catch (Exception e) { + throw new RuntimeException("Couldn't initialize a HttpClusterStateProvider (is/are the " + + "Solr server(s), " + solrUrls + ", down?)", e); + } + } else { + throw new IllegalArgumentException("Both zkHosts and solrUrl cannot be null."); + } } else { this.stateProvider = stateProvider; } @@ -1259,7 +1279,7 @@ public class CloudSolrClient extends SolrClient { Set liveNodes = stateProvider.liveNodes(); for (String liveNode : liveNodes) { theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode, - (String) stateProvider.getClusterProperties().getOrDefault(ZkStateReader.URL_SCHEME,"http"))); + (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http"))); } } else { @@ -1365,7 +1385,7 @@ public class CloudSolrClient extends SolrClient { return rsp.getResponse(); } - Set getCollectionNames(String collection) { + private Set getCollectionNames(String collection) { // Extract each comma separated collection name and store in a List. List rawCollectionsList = StrUtils.splitSmart(collection, ",", true); Set collectionNames = new HashSet<>(); @@ -1602,6 +1622,7 @@ public class CloudSolrClient extends SolrClient { */ public static class Builder { private Collection zkHosts; + private List solrUrls; private HttpClient httpClient; private String zkChroot; private LBHttpSolrClient loadBalancedSolrClient; @@ -1613,6 +1634,7 @@ public class CloudSolrClient extends SolrClient { public Builder() { this.zkHosts = new ArrayList(); + this.solrUrls = new ArrayList(); this.shardLeadersOnly = true; } @@ -1629,7 +1651,28 @@ public class CloudSolrClient extends SolrClient { this.zkHosts.add(zkHost); return this; } + + /** + * Provide a Solr URL to be used when configuring {@link CloudSolrClient} instances. + * + * Method may be called multiple times. One of the provided values will be used to fetch + * the list of live Solr nodes that the underlying {@link HttpClusterStateProvider} would be maintaining. + */ + public Builder withSolrUrl(String solrUrl) { + this.solrUrls.add(solrUrl); + return this; + } + /** + * Provide a list of Solr URL to be used when configuring {@link CloudSolrClient} instances. + * One of the provided values will be used to fetch the list of live Solr + * nodes that the underlying {@link HttpClusterStateProvider} would be maintaining. + */ + public Builder withSolrUrl(Collection solrUrls) { + this.solrUrls.addAll(solrUrls); + return this; + } + /** * Provides a {@link HttpClient} for the builder to use when creating clients. */ @@ -1722,24 +1765,51 @@ public class CloudSolrClient extends SolrClient { */ public CloudSolrClient build() { if (stateProvider == null) { - stateProvider = new ZkClientClusterStateProvider(zkHosts, zkChroot); + if (!zkHosts.isEmpty()) { + stateProvider = new ZkClientClusterStateProvider(zkHosts, zkChroot); + } + else if (!this.solrUrls.isEmpty()) { + try { + stateProvider = new HttpClusterStateProvider(solrUrls, httpClient); + } catch (Exception e) { + throw new RuntimeException("Couldn't initialize a HttpClusterStateProvider (is/are the " + + "Solr server(s), " + solrUrls + ", down?)", e); + } + } else { + throw new IllegalArgumentException("Both zkHosts and solrUrl cannot be null."); + } } - return new CloudSolrClient(zkHosts, zkChroot, httpClient, loadBalancedSolrClient, lbClientBuilder, + return new CloudSolrClient(zkHosts, zkChroot, solrUrls, httpClient, loadBalancedSolrClient, lbClientBuilder, shardLeadersOnly, directUpdatesToLeadersOnly, stateProvider); } } interface ClusterStateProvider extends Closeable { + /** + * Obtain the state of the collection (cluster status). + * @return the collection state, or null is collection doesn't exist + */ ClusterState.CollectionRef getState(String collection); + /** + * Obtain set of live_nodes for the cluster. + */ Set liveNodes(); String getAlias(String collection); String getCollectionName(String name); - Map getClusterProperties(); + /** + * Obtain a cluster property, or null if it doesn't exist. + */ + Object getClusterProperty(String propertyName); + + /** + * Obtain a cluster property, or the default value if it doesn't exist. + */ + Object getClusterProperty(String propertyName, String def); void connect(); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java new file mode 100644 index 00000000000..b5cf41444b6 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.impl; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.http.client.HttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ClusterState.CollectionRef; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HttpClusterStateProvider implements CloudSolrClient.ClusterStateProvider { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private String urlScheme; + volatile Set liveNodes; + long liveNodesTimestamp = 0; + volatile Map aliases; + long aliasesTimestamp = 0; + + private int cacheTimeout = 5; // the liveNodes and aliases cache will be invalidated after 5 secs + final HttpClient httpClient; + final boolean clientIsInternal; + + public HttpClusterStateProvider(List solrUrls, HttpClient httpClient) throws Exception { + this.httpClient = httpClient == null? HttpClientUtil.createClient(null): httpClient; + this.clientIsInternal = httpClient == null; + for (String solrUrl: solrUrls) { + urlScheme = solrUrl.startsWith("https")? "https": "http"; + try (SolrClient initialClient = new HttpSolrClient.Builder().withBaseSolrUrl(solrUrl).withHttpClient(httpClient).build()) { + Set liveNodes = fetchLiveNodes(initialClient); // throws exception if unable to fetch + this.liveNodes = liveNodes; + liveNodesTimestamp = System.nanoTime(); + break; + } catch (IOException e) { + log.warn("Attempt to fetch live_nodes from " + solrUrl + " failed.", e); + } + } + + if (this.liveNodes == null || this.liveNodes.isEmpty()) { + throw new RuntimeException("Tried fetching live_nodes using Solr URLs provided, i.e. " + solrUrls + ". However, " + + "succeeded in obtaining the cluster state from none of them." + + "If you think your Solr cluster is up and is accessible," + + " you could try re-creating a new CloudSolrClient using a working" + + " solrUrl or zkUrl."); + } + } + + @Override + public void close() throws IOException { + if (this.clientIsInternal && this.httpClient != null) { + HttpClientUtil.close(httpClient); + } + } + + @Override + public CollectionRef getState(String collection) { + for (String nodeName: liveNodes) { + try (HttpSolrClient client = new HttpSolrClient.Builder(). + withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)). + withHttpClient(httpClient).build()) { + ClusterState cs = fetchClusterState(client, collection); + return cs.getCollectionRef(collection); + } catch (SolrServerException | RemoteSolrException | IOException e) { + if (e.getMessage().contains(collection + " not found")) { + // Cluster state for the given collection was not found. + // Lets fetch/update our aliases: + getAliases(true); + return null; + } + log.warn("Attempt to fetch cluster state from " + + ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme) + " failed.", e); + } + } + throw new RuntimeException("Tried fetching cluster state using the node names we knew of, i.e. " + liveNodes +". However, " + + "succeeded in obtaining the cluster state from none of them." + + "If you think your Solr cluster is up and is accessible," + + " you could try re-creating a new CloudSolrClient using a working" + + " solrUrl or zkUrl."); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private ClusterState fetchClusterState(SolrClient client, String collection) throws SolrServerException, IOException { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("collection", collection); + params.set("action", "CLUSTERSTATUS"); + QueryRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + NamedList cluster = (SimpleOrderedMap) client.request(request).get("cluster"); + Map collectionsMap = ((NamedList) cluster.get("collections")).asShallowMap(); + int znodeVersion = (int)((Map)(collectionsMap).get(collection)).get("znodeVersion"); + Set liveNodes = new HashSet((List)(cluster.get("live_nodes"))); + this.liveNodes = liveNodes; + liveNodesTimestamp = System.nanoTime(); + ClusterState cs = ClusterState.load(znodeVersion, collectionsMap, liveNodes, ZkStateReader.CLUSTER_STATE); + return cs; + } + + @Override + public Set liveNodes() { + if (liveNodes == null) { + throw new RuntimeException("We don't know of any live_nodes to fetch the" + + " latest live_nodes information from. " + + "If you think your Solr cluster is up and is accessible," + + " you could try re-creating a new CloudSolrClient using a working" + + " solrUrl or zkUrl."); + } + if (TimeUnit.SECONDS.convert((System.nanoTime() - liveNodesTimestamp), TimeUnit.NANOSECONDS) > getCacheTimeout()) { + for (String nodeName: liveNodes) { + try (HttpSolrClient client = new HttpSolrClient.Builder(). + withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)). + withHttpClient(httpClient).build()) { + Set liveNodes = fetchLiveNodes(client); + this.liveNodes = (liveNodes); + liveNodesTimestamp = System.nanoTime(); + return liveNodes; + } catch (Exception e) { + log.warn("Attempt to fetch live_nodes from " + + ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme) + " failed.", e); + } + } + throw new RuntimeException("Tried fetching live_nodes using all the node names we knew of, i.e. " + liveNodes +". However, " + + "succeeded in obtaining the cluster state from none of them." + + "If you think your Solr cluster is up and is accessible," + + " you could try re-creating a new CloudSolrClient using a working" + + " solrUrl or zkUrl."); + } else { + return liveNodes; // cached copy is fresh enough + } + } + + private static Set fetchLiveNodes(SolrClient client) throws Exception { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", "CLUSTERSTATUS"); + QueryRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + NamedList cluster = (SimpleOrderedMap) client.request(request).get("cluster"); + Set liveNodes = new HashSet((List)(cluster.get("live_nodes"))); + return liveNodes; + } + + @Override + public String getAlias(String collection) { + Map aliases = getAliases(false); + return aliases.get(collection); + } + + private Map getAliases(boolean forceFetch) { + if (this.liveNodes == null) { + throw new RuntimeException("We don't know of any live_nodes to fetch the" + + " latest aliases information from. " + + "If you think your Solr cluster is up and is accessible," + + " you could try re-creating a new CloudSolrClient using a working" + + " solrUrl or zkUrl."); + } + + if (forceFetch || this.aliases == null || + TimeUnit.SECONDS.convert((System.nanoTime() - aliasesTimestamp), TimeUnit.NANOSECONDS) > getCacheTimeout()) { + for (String nodeName: liveNodes) { + try (HttpSolrClient client = new HttpSolrClient.Builder(). + withBaseSolrUrl(ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme)). + withHttpClient(httpClient).build()) { + + Map aliases = new CollectionAdminRequest.ListAliases().process(client).getAliases(); + this.aliases = aliases; + this.aliasesTimestamp = System.nanoTime(); + return Collections.unmodifiableMap(aliases); + } catch (SolrServerException | RemoteSolrException | IOException e) { + log.warn("Attempt to fetch cluster state from " + + ZkStateReader.getBaseUrlForNodeName(nodeName, urlScheme) + " failed.", e); + } + } + + throw new RuntimeException("Tried fetching aliases using all the node names we knew of, i.e. " + liveNodes +". However, " + + "succeeded in obtaining the cluster state from none of them." + + "If you think your Solr cluster is up and is accessible," + + " you could try re-creating a new CloudSolrClient using a working" + + " solrUrl or zkUrl."); + } else { + return Collections.unmodifiableMap(this.aliases); // cached copy is fresh enough + } + } + + @Override + public String getCollectionName(String name) { + Map aliases = getAliases(false); + return aliases.containsKey(name) ? aliases.get(name): name; + } + + @Override + public Object getClusterProperty(String propertyName) { + if (propertyName.equals(ZkStateReader.URL_SCHEME)) { + return this.urlScheme; + } + throw new UnsupportedOperationException("Fetching cluster properties not supported" + + " using the HttpClusterStateProvider. " + + "ZkClientClusterStateProvider can be used for this."); // TODO + } + + @Override + public Object getClusterProperty(String propertyName, String def) { + if (propertyName.equals(ZkStateReader.URL_SCHEME)) { + return this.urlScheme; + } + throw new UnsupportedOperationException("Fetching cluster properties not supported" + + " using the HttpClusterStateProvider. " + + "ZkClientClusterStateProvider can be used for this."); // TODO + } + + @Override + public void connect() {} + + public int getCacheTimeout() { + return cacheTimeout; + } + + public void setCacheTimeout(int cacheTimeout) { + this.cacheTimeout = cacheTimeout; + } + +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java index 8ed1b5c45aa..6b37eb71134 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java @@ -69,8 +69,18 @@ public class ZkClientClusterStateProvider implements CloudSolrClient.ClusterStat } @Override - public Map getClusterProperties() { - return zkStateReader.getClusterProperties(); + public Object getClusterProperty(String propertyName) { + Map props = zkStateReader.getClusterProperties(); + return props.get(propertyName); + } + + @Override + public Object getClusterProperty(String propertyName, String def) { + Map props = zkStateReader.getClusterProperties(); + if (props.containsKey(propertyName)) { + return props.get(propertyName); + } + return def; } @Override diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index f87f149f9b7..ec43e11f7cb 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -2246,6 +2246,20 @@ public abstract class CollectionAdminRequest } + // LISTALIASES request + public static class ListAliases extends CollectionAdminRequest { + + public ListAliases() { + super(CollectionAction.LISTALIASES); + } + + @Override + protected CollectionAdminResponse createResponse(SolrClient client) { + return new CollectionAdminResponse(); + } + + } + /** * Returns a SolrRequest to get a list of collections in the cluster */ diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java index 82d4d6f06d9..6821075b366 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java @@ -16,6 +16,7 @@ */ package org.apache.solr.client.solrj.response; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -60,6 +61,16 @@ public class CollectionAdminResponse extends SolrResponseBase return res; } + @SuppressWarnings("unchecked") + public Map getAliases() + { + NamedList response = getResponse(); + if (response.get("aliases") != null) { + return ((Map)response.get("aliases")); + } + return Collections.emptyMap(); + } + @SuppressWarnings("unchecked") public Map> getCollectionNodesStatus() { diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java index 302ee62e434..65bd81b80d4 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java @@ -322,6 +322,10 @@ public class ClusterState implements JSONWriter.Writable { return new ClusterState(version, liveNodes, Collections.emptyMap()); } Map stateMap = (Map) Utils.fromJSON(bytes); + return load(version, stateMap, liveNodes, znode); + } + + public static ClusterState load(Integer version, Map stateMap, Set liveNodes, String znode) { Map collections = new LinkedHashMap<>(stateMap.size()); for (Entry entry : stateMap.entrySet()) { String collectionName = entry.getKey(); @@ -332,7 +336,6 @@ public class ClusterState implements JSONWriter.Writable { return new ClusterState( liveNodes, collections,version); } - public static Aliases load(byte[] bytes) { if (bytes == null || bytes.length == 0) { return new Aliases(); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index 51db039ea34..d79fafa3efd 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -68,6 +68,7 @@ public interface CollectionParams { SYNCSHARD(true, LockLevel.SHARD), CREATEALIAS(true, LockLevel.COLLECTION), DELETEALIAS(true, LockLevel.COLLECTION), + LISTALIASES(false, LockLevel.NONE), SPLITSHARD(true, LockLevel.SHARD), DELETESHARD(true, LockLevel.SHARD), CREATESHARD(true, LockLevel.COLLECTION), diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java index d260b02da99..d94d7e414a2 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java @@ -130,11 +130,6 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 { return livenodes; } - @Override - public Map getClusterProperties() { - return Collections.EMPTY_MAP; - } - @Override public String getAlias(String collection) { return collection; @@ -152,6 +147,16 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 { public void close() throws IOException { } + + @Override + public Object getClusterProperty(String propertyName) { + return null; + } + + @Override + public Object getClusterProperty(String propertyName, String def) { + return def; + } }; } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java index 5ebb650c05d..c91cb67a2fa 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java @@ -68,6 +68,7 @@ import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.admin.ConfigSetsHandler; import org.apache.solr.handler.admin.CoreAdminHandler; +import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; @@ -90,6 +91,8 @@ public class CloudSolrClientTest extends SolrCloudTestCase { private static final int TIMEOUT = 30; private static final int NODE_COUNT = 3; + private static CloudSolrClient httpBasedCloudSolrClient = null; + @BeforeClass public static void setupCluster() throws Exception { configureCluster(NODE_COUNT) @@ -99,8 +102,21 @@ public class CloudSolrClientTest extends SolrCloudTestCase { CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1).process(cluster.getSolrClient()); AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(), false, true, TIMEOUT); + + httpBasedCloudSolrClient = new CloudSolrClient.Builder().withSolrUrl( + cluster.getJettySolrRunner(0).getBaseUrl().toString()).build(); } + @AfterClass + public static void afterClass() { + if (httpBasedCloudSolrClient != null) { + try { + httpBasedCloudSolrClient.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Before @@ -110,6 +126,13 @@ public class CloudSolrClientTest extends SolrCloudTestCase { .commit(cluster.getSolrClient(), COLLECTION); } + /** + * Randomly return the cluster's ZK based CSC, or HttpClusterProvider based CSC. + */ + private CloudSolrClient getRandomClient() { + return random().nextBoolean()? cluster.getSolrClient(): httpBasedCloudSolrClient; + } + @Test public void testParallelUpdateQTime() throws Exception { UpdateRequest req = new UpdateRequest(); @@ -118,7 +141,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase { doc.addField("id", String.valueOf(TestUtil.nextInt(random(), 1000, 1100))); req.add(doc); } - UpdateResponse response = req.process(cluster.getSolrClient(), COLLECTION); + UpdateResponse response = req.process(getRandomClient(), COLLECTION); // See SOLR-6547, we just need to ensure that no exception is thrown here assertTrue(response.getQTime() >= 0); } @@ -143,33 +166,48 @@ public class CloudSolrClientTest extends SolrCloudTestCase { .add(new SolrInputDocument(id, "1", "a_t", "hello2"), false) .commit(cluster.getSolrClient(), "overwrite"); - resp = cluster.getSolrClient().query("overwrite", new SolrQuery("*:*")); + resp = getRandomClient().query("overwrite", new SolrQuery("*:*")); assertEquals("There should be 3 documents because there should be two id=1 docs due to overwrite=false", 3, resp.getResults().getNumFound()); } + @Test + public void testAliasHandling() throws Exception { + CloudSolrClient client = getRandomClient(); + SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc"); + client.add(COLLECTION, doc); + client.commit(COLLECTION); + + CollectionAdminRequest.createAlias("testalias", COLLECTION).process(cluster.getSolrClient()); + + // ensure that the alias has been registered + assertEquals(COLLECTION, + new CollectionAdminRequest.ListAliases().process(cluster.getSolrClient()).getAliases().get("testalias")); + + assertEquals(1, client.query(COLLECTION, params("q", "*:*")).getResults().getNumFound()); + assertEquals(1, client.query("testalias", params("q", "*:*")).getResults().getNumFound()); + } + @Test public void testHandlingOfStaleAlias() throws Exception { - try (CloudSolrClient client = getCloudSolrClient(cluster.getZkServer().getZkAddress())) { - client.setDefaultCollection("misconfigured-alias"); + CloudSolrClient client = getRandomClient(); - CollectionAdminRequest.createCollection("nemesis", "conf", 2, 1).process(client); - CollectionAdminRequest.createAlias("misconfigured-alias", "nemesis").process(client); - CollectionAdminRequest.deleteCollection("nemesis").process(client); + CollectionAdminRequest.createCollection("nemesis", "conf", 2, 1).process(client); + CollectionAdminRequest.createAlias("misconfigured-alias", "nemesis").process(client); + CollectionAdminRequest.deleteCollection("nemesis").process(client); - List docs = new ArrayList<>(); + List docs = new ArrayList<>(); - SolrInputDocument doc = new SolrInputDocument(); - doc.addField(id, Integer.toString(1)); - docs.add(doc); + SolrInputDocument doc = new SolrInputDocument(); + doc.addField(id, Integer.toString(1)); + docs.add(doc); - try { - client.add(docs); - fail("Alias points to non-existing collection, add should fail"); - } catch (SolrException e) { - assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); - assertTrue("Unexpected exception", e.getMessage().contains("Collection not found")); - } + try { + client.add("misconfigured-alias", docs); + fail("Alias points to non-existing collection, add should fail"); + } catch (SolrException e) { + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue("Unexpected exception", e.getMessage().contains("Collection not found")); } } @@ -182,8 +220,8 @@ public class CloudSolrClientTest extends SolrCloudTestCase { .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); // Test single threaded routed updates for UpdateRequest - NamedList response = cluster.getSolrClient().request(request, COLLECTION); - if (cluster.getSolrClient().isDirectUpdatesToLeadersOnly()) { + NamedList response = getRandomClient().request(request, COLLECTION); + if (getRandomClient().isDirectUpdatesToLeadersOnly()) { checkSingleServer(response); } CloudSolrClient.RouteResponse rr = (CloudSolrClient.RouteResponse) response; @@ -214,11 +252,11 @@ public class CloudSolrClientTest extends SolrCloudTestCase { .deleteById("0") .deleteById("2") .commit(cluster.getSolrClient(), COLLECTION); - if (cluster.getSolrClient().isDirectUpdatesToLeadersOnly()) { + if (getRandomClient().isDirectUpdatesToLeadersOnly()) { checkSingleServer(uResponse.getResponse()); } - QueryResponse qResponse = cluster.getSolrClient().query(COLLECTION, new SolrQuery("*:*")); + QueryResponse qResponse = getRandomClient().query(COLLECTION, new SolrQuery("*:*")); SolrDocumentList docs = qResponse.getResults(); assertEquals(0, docs.getNumFound()); @@ -307,7 +345,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase { ModifiableSolrParams solrParams = new ModifiableSolrParams(); solrParams.set(CommonParams.Q, "*:*"); solrParams.set(ShardParams._ROUTE_, sameShardRoutes.get(random().nextInt(sameShardRoutes.size()))); - log.info("output: {}", cluster.getSolrClient().query(COLLECTION, solrParams)); + log.info("output: {}", getRandomClient().query(COLLECTION, solrParams)); } // Request counts increase from expected nodes should aggregate to 1000, while there should be @@ -362,10 +400,10 @@ public class CloudSolrClientTest extends SolrCloudTestCase { .add(id, "0", "a_t", "hello1") .add(id, "2", "a_t", "hello2") .add(id, "3", "a_t", "hello2") - .commit(cluster.getSolrClient(), collectionName); + .commit(getRandomClient(), collectionName); // Run the actual test for 'preferLocalShards' - queryWithPreferLocalShards(cluster.getSolrClient(), true, collectionName); + queryWithPreferLocalShards(getRandomClient(), true, collectionName); } private void queryWithPreferLocalShards(CloudSolrClient cloudClient, @@ -658,7 +696,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase { .add("id", "2", "a_t", "hello2"); updateRequest.setParam(UpdateParams.VERSIONS, Boolean.TRUE.toString()); - NamedList response = updateRequest.commit(cluster.getSolrClient(), COLLECTION).getResponse(); + NamedList response = updateRequest.commit(getRandomClient(), COLLECTION).getResponse(); Object addsObject = response.get("adds"); assertNotNull("There must be a adds parameter", addsObject); @@ -677,7 +715,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase { assertTrue("Version for id 2 must be a long", object instanceof Long); versions.put("2", (Long) object); - QueryResponse resp = cluster.getSolrClient().query(COLLECTION, new SolrQuery("*:*")); + QueryResponse resp = getRandomClient().query(COLLECTION, new SolrQuery("*:*")); assertEquals("There should be one document because overwrite=true", 2, resp.getResults().getNumFound()); for (SolrDocument doc : resp.getResults()) { @@ -688,13 +726,38 @@ public class CloudSolrClientTest extends SolrCloudTestCase { // assert that "deletes" are returned UpdateRequest deleteRequest = new UpdateRequest().deleteById("1"); deleteRequest.setParam(UpdateParams.VERSIONS, Boolean.TRUE.toString()); - response = deleteRequest.commit(cluster.getSolrClient(), COLLECTION).getResponse(); + response = deleteRequest.commit(getRandomClient(), COLLECTION).getResponse(); Object deletesObject = response.get("deletes"); assertNotNull("There must be a deletes parameter", deletesObject); NamedList deletes = (NamedList) deletesObject; assertEquals("There must be 1 version", 1, deletes.size()); } + @Test + public void testInitializationWithSolrUrls() throws Exception { + CloudSolrClient client = httpBasedCloudSolrClient; + SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc"); + client.add(COLLECTION, doc); + client.commit(COLLECTION); + assertEquals(1, client.query(COLLECTION, params("q", "*:*")).getResults().getNumFound()); + } + + @Test + public void testCollectionDoesntExist() throws Exception { + CloudSolrClient client = getRandomClient(); + SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc"); + try { + client.add("boguscollectionname", doc); + fail(); + } catch (SolrException ex) { + if (ex.getMessage().equals("Collection not found: boguscollectionname")) { + // pass + } else { + throw ex; + } + } + } + private static void checkSingleServer(NamedList response) { final CloudSolrClient.RouteResponse rr = (CloudSolrClient.RouteResponse) response; final Map routes = rr.getRoutes();