From 38d16dc7b764d1548b16808fd8e5609665140272 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Thu, 13 Dec 2012 03:21:11 +0000 Subject: [PATCH] SOLR-4140: Allow access to the collections API through CloudSolrServer without referencing an existing collection. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1421071 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 + .../solr/cloud/BasicDistributedZkTest.java | 64 +++++-- .../client/solrj/impl/CloudSolrServer.java | 173 ++++++++++-------- .../cloud/AbstractFullDistribZkTestBase.java | 20 +- 4 files changed, 161 insertions(+), 99 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index f9a94d950d4..8529d47a4c5 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -143,6 +143,9 @@ New Features * SOLR-4166: LBHttpSolrServer ignores ResponseParser passed in constructor. (Steve Molloy via Mark Miller) +* SOLR-4140: Allow access to the collections API through CloudSolrServer + without referencing an existing collection. (Per Steffensen via Mark Miller) + Optimizations ---------------------- diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 759e8b2cebe..55e68aa7da4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -748,8 +748,37 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase { for (int i = 0; i < cnt; i++) { int numShards = _TestUtil.nextInt(random(), 0, shardCount) + 1; int replicationFactor = _TestUtil.nextInt(random(), 0, 3) + 2; - int maxShardsPerNode = (((numShards * replicationFactor) / getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size())) + 1; - createCollection(collectionInfos, i, numShards, replicationFactor, maxShardsPerNode); + int maxShardsPerNode = (((numShards * replicationFactor) / getCommonCloudSolrServer() + .getZkStateReader().getClusterState().getLiveNodes().size())) + 1; + + + CloudSolrServer client = null; + try { + if (i == 0) { + // Test if we can create a collection through CloudSolrServer where + // you havnt set default-collection + // This is nice because you want to be able to create you first + // collection using CloudSolrServer, and in such case there is + // nothing reasonable to set as default-collection + client = createCloudClient(null); + } else if (i == 1) { + // Test if we can create a collection through CloudSolrServer where + // you have set default-collection to a non-existing collection + // This is nice because you want to be able to create you first + // collection using CloudSolrServer, and in such case there is + // nothing reasonable to set as default-collection, but you might want + // to use the same CloudSolrServer throughout the entire + // lifetime of your client-application, so it is nice to be able to + // set a default-collection on this CloudSolrServer once and for all + // and use this CloudSolrServer to create the collection + client = createCloudClient("awholynewcollection_" + i); + } + + createCollection(collectionInfos, "awholynewcollection_" + i, + numShards, replicationFactor, maxShardsPerNode, client); + } finally { + if (client != null) client.shutdown(); + } } Set>> collectionInfosEntrySet = collectionInfos.entrySet(); @@ -882,7 +911,12 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase { int replicationFactor = 2; int maxShardsPerNode = 1; collectionInfos = new HashMap>(); - createCollection(collectionInfos, cnt, numShards, replicationFactor, maxShardsPerNode); + CloudSolrServer client = createCloudClient("awholynewcollection_" + cnt); + try { + createCollection(collectionInfos, "awholynewcollection_" + cnt, numShards, replicationFactor, maxShardsPerNode, client); + } finally { + client.shutdown(); + } // TODO: REMOVE THE SLEEP IN THE METHOD CALL WHEN WE HAVE COLLECTION API // RESPONSES @@ -893,30 +927,34 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase { protected void createCollection(Map> collectionInfos, - int i, int numShards, int numReplica, int maxShardsPerNode) throws SolrServerException, IOException { + String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client) throws SolrServerException, IOException { ModifiableSolrParams params = new ModifiableSolrParams(); params.set("action", CollectionAction.CREATE.toString()); params.set(OverseerCollectionProcessor.NUM_SLICES, numShards); - params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplica); + params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas); params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode); - String collectionName = "awholynewcollection_" + i; + int clientIndex = random().nextInt(2); List list = new ArrayList(); list.add(numShards); - list.add(numReplica); + list.add(numReplicas); list.add(maxShardsPerNode); collectionInfos.put(collectionName, list); params.set("name", collectionName); SolrRequest request = new QueryRequest(params); request.setPath("/admin/collections"); - final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring( - 0, - ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length() - - DEFAULT_COLLECTION.length() - 1); - - createNewSolrServer("", baseUrl).request(request); + if (client == null) { + final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring( + 0, + ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length() + - DEFAULT_COLLECTION.length() - 1); + + createNewSolrServer("", baseUrl).request(request); + } else { + client.request(request); + } } private boolean waitForReloads(String collectionName, Map urlToTimeBefore) throws SolrServerException, IOException { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java index 5e05e0a3b08..b70349f8b47 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java @@ -163,9 +163,10 @@ public class CloudSolrServer extends SolrServer { } @Override - public NamedList request(SolrRequest request) throws SolrServerException, IOException { + public NamedList request(SolrRequest request) + throws SolrServerException, IOException { connect(); - + // TODO: if you can hash here, you could favor the shard leader ClusterState clusterState = zkStateReader.getClusterState(); @@ -176,95 +177,111 @@ public class CloudSolrServer extends SolrServer { sendToLeaders = true; replicas = new ArrayList(); } - + SolrParams reqParams = request.getParams(); if (reqParams == null) { reqParams = new ModifiableSolrParams(); } - String collection = reqParams.get("collection", defaultCollection); - - if (collection == null) { - throw new SolrServerException("No collection param specified on request and no default collection has been set."); - } - - // Extract each comma separated collection name and store in a List. - List collectionList = StrUtils.splitSmart(collection, ",", true); - - // TODO: not a big deal because of the caching, but we could avoid looking at every shard - // when getting leaders if we tweaked some things - - // Retrieve slices from the cloud state and, for each collection specified, - // add it to the Map of slices. - Map slices = new HashMap(); - for (String collectionName : collectionList) { - ClientUtils.addSlices(slices, collectionName, clusterState.getSlices(collectionName), true); - } - - Set liveNodes = clusterState.getLiveNodes(); - - List theUrlList; - synchronized (cachLock) { - List leaderUrlList = leaderUrlLists.get(collection); - List urlList = urlLists.get(collection); - List replicasList = replicasLists.get(collection); - - if ((sendToLeaders && leaderUrlList == null) || (!sendToLeaders - && urlList == null) - || clusterState.hashCode() != this.lastClusterStateHashCode) { - // build a map of unique nodes - // TODO: allow filtering by group, role, etc - Map nodes = new HashMap(); - List urlList2 = new ArrayList(); - for (Slice slice : slices.values()) { - for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) { - ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps); - String node = coreNodeProps.getNodeName(); - if (!liveNodes.contains(coreNodeProps.getNodeName()) - || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue; - if (nodes.put(node, nodeProps) == null) { - if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) { - String url = coreNodeProps.getCoreUrl(); - urlList2.add(url); - } else if (sendToLeaders) { - String url = coreNodeProps.getCoreUrl(); - replicas.add(url); + List theUrlList = new ArrayList(); + if (request.getPath().equals("/admin/collections")) { + Set liveNodes = clusterState.getLiveNodes(); + for (String liveNode : liveNodes) { + int splitPointBetweenHostPortAndContext = liveNode.indexOf("_"); + theUrlList.add("http://" + + liveNode.substring(0, splitPointBetweenHostPortAndContext) + "/" + + liveNode.substring(splitPointBetweenHostPortAndContext + 1)); + } + } else { + String collection = reqParams.get("collection", defaultCollection); + + if (collection == null) { + throw new SolrServerException( + "No collection param specified on request and no default collection has been set."); + } + + // Extract each comma separated collection name and store in a List. + List collectionList = StrUtils.splitSmart(collection, ",", true); + + // TODO: not a big deal because of the caching, but we could avoid looking + // at every shard + // when getting leaders if we tweaked some things + + // Retrieve slices from the cloud state and, for each collection + // specified, + // add it to the Map of slices. + Map slices = new HashMap(); + for (String collectionName : collectionList) { + ClientUtils.addSlices(slices, collectionName, + clusterState.getSlices(collectionName), true); + } + Set liveNodes = clusterState.getLiveNodes(); + + synchronized (cachLock) { + List leaderUrlList = leaderUrlLists.get(collection); + List urlList = urlLists.get(collection); + List replicasList = replicasLists.get(collection); + + if ((sendToLeaders && leaderUrlList == null) + || (!sendToLeaders && urlList == null) + || clusterState.hashCode() != this.lastClusterStateHashCode) { + // build a map of unique nodes + // TODO: allow filtering by group, role, etc + Map nodes = new HashMap(); + List urlList2 = new ArrayList(); + for (Slice slice : slices.values()) { + for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) { + ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps); + String node = coreNodeProps.getNodeName(); + if (!liveNodes.contains(coreNodeProps.getNodeName()) + || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue; + if (nodes.put(node, nodeProps) == null) { + if (!sendToLeaders + || (sendToLeaders && coreNodeProps.isLeader())) { + String url = coreNodeProps.getCoreUrl(); + urlList2.add(url); + } else if (sendToLeaders) { + String url = coreNodeProps.getCoreUrl(); + replicas.add(url); + } } } } + + if (sendToLeaders) { + this.leaderUrlLists.put(collection, urlList2); + leaderUrlList = urlList2; + this.replicasLists.put(collection, replicas); + replicasList = replicas; + } else { + this.urlLists.put(collection, urlList2); + urlList = urlList2; + } + this.lastClusterStateHashCode = clusterState.hashCode(); } + if (sendToLeaders) { - this.leaderUrlLists.put(collection, urlList2); - leaderUrlList = urlList2; - this.replicasLists.put(collection, replicas); - replicasList = replicas; + theUrlList = new ArrayList(leaderUrlList.size()); + theUrlList.addAll(leaderUrlList); } else { - this.urlLists.put(collection, urlList2); - urlList = urlList2; + theUrlList = new ArrayList(urlList.size()); + theUrlList.addAll(urlList); + } + Collections.shuffle(theUrlList, rand); + if (sendToLeaders) { + ArrayList theReplicas = new ArrayList( + replicasList.size()); + theReplicas.addAll(replicasList); + Collections.shuffle(theReplicas, rand); + // System.out.println("leaders:" + theUrlList); + // System.out.println("replicas:" + theReplicas); + theUrlList.addAll(theReplicas); } - this.lastClusterStateHashCode = clusterState.hashCode(); - } - - if (sendToLeaders) { - theUrlList = new ArrayList(leaderUrlList.size()); - theUrlList.addAll(leaderUrlList); - } else { - theUrlList = new ArrayList(urlList.size()); - theUrlList.addAll(urlList); - } - Collections.shuffle(theUrlList, rand); - if (sendToLeaders) { - ArrayList theReplicas = new ArrayList( - replicasList.size()); - theReplicas.addAll(replicasList); - Collections.shuffle(theReplicas, rand); - // System.out.println("leaders:" + theUrlList); - // System.out.println("replicas:" + theReplicas); - theUrlList.addAll(theReplicas); } } - - // System.out.println("########################## MAKING REQUEST TO " + theUrlList); - + + // System.out.println("########################## MAKING REQUEST TO " + + // theUrlList); + LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, theUrlList); LBHttpSolrServer.Rsp rsp = lbServer.request(req); return rsp.getResponse(); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index dcd4f6c7ab5..e98efa8d783 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -199,13 +199,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes assert(cloudInit == false); cloudInit = true; try { - CloudSolrServer server = new CloudSolrServer(zkServer.getZkAddress()); - server.setDefaultCollection(DEFAULT_COLLECTION); - server.getLbServer().getHttpClient().getParams() - .setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 5000); - server.getLbServer().getHttpClient().getParams() - .setParameter(CoreConnectionPNames.SO_TIMEOUT, 20000); - cloudClient = server; + cloudClient = createCloudClient(DEFAULT_COLLECTION); cloudClient.connect(); } catch (MalformedURLException e) { @@ -217,7 +211,17 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes chaosMonkey = new ChaosMonkey(zkServer, zkStateReader, DEFAULT_COLLECTION, shardToJetty, shardToLeaderJetty); } - + + protected CloudSolrServer createCloudClient(String defaultCollection) + throws MalformedURLException { + CloudSolrServer server = new CloudSolrServer(zkServer.getZkAddress()); + if (defaultCollection != null) server.setDefaultCollection(defaultCollection); + server.getLbServer().getHttpClient().getParams() + .setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 5000); + server.getLbServer().getHttpClient().getParams() + .setParameter(CoreConnectionPNames.SO_TIMEOUT, 20000); + return server; + } @Override protected void createServers(int numServers) throws Exception {