From 5952539b073aae659c038f217f94dd1e83376d37 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Fri, 1 Nov 2013 00:50:08 +0000 Subject: [PATCH] SOLR-5392: Extend solrj apis to cover collection management. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1537787 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 +- .../CollectionsAPIDistributedZkTest.java | 84 ++++ .../solrj/request/CollectionAdminRequest.java | 402 ++++++++++++++++++ .../response/CollectionAdminResponse.java | 79 ++++ 4 files changed, 567 insertions(+), 1 deletion(-) create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 51c1e457fde..b6e72327004 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -119,7 +119,8 @@ New Features via the optional DocBasedVersionConstraintsProcessorFactory update processor (Hossman, yonik) - +* SOLR-5392: Extend solrj apis to cover collection management. + (Roman Shaposhnik via Mark Miller) Bug Fixes ---------------------- diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java index b0a059dae23..f73b118d672 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java @@ -48,14 +48,17 @@ import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.util._TestUtil; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.CloudSolrServer; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.CoreAdminResponse; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; @@ -189,6 +192,7 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa @Override public void doTest() throws Exception { + testSolrJAPICalls(); testNodesUsedByCreate(); testCollectionsAPI(); testErrorHandling(); @@ -235,6 +239,86 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa } + private void testSolrJAPICalls() throws Exception { + SolrServer server = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))); + CollectionAdminResponse response; + Map> coresStatus; + Map> nodesStatus; + + response = CollectionAdminRequest.createCollection("solrj_collection", + 2, 2, null, + null, "conf1", "myOwnField", + server); + assertEquals(0, response.getStatus()); + assertTrue(response.isSuccess()); + coresStatus = response.getCollectionCoresStatus(); + assertEquals(4, coresStatus.size()); + for (int i=0; i<4; i++) { + NamedList status = coresStatus.get("solrj_collection_shard" + (i/2+1) + "_replica" + (i%2+1)); + assertEquals(0, (int)status.get("status")); + assertTrue(status.get("QTime") > 0); + } + + response = CollectionAdminRequest.createCollection("solrj_implicit", + "shardA,shardB", server); + assertEquals(0, response.getStatus()); + assertTrue(response.isSuccess()); + coresStatus = response.getCollectionCoresStatus(); + assertEquals(2, coresStatus.size()); + + response = CollectionAdminRequest.createShard("solrj_implicit", "shardC", server); + assertEquals(0, response.getStatus()); + assertTrue(response.isSuccess()); + coresStatus = response.getCollectionCoresStatus(); + assertEquals(1, coresStatus.size()); + assertEquals(0, (int) coresStatus.get("solrj_implicit_shardC_replica1").get("status")); + + response = CollectionAdminRequest.deleteShard("solrj_implicit", "shardC", server); + assertEquals(0, response.getStatus()); + assertTrue(response.isSuccess()); + nodesStatus = response.getCollectionNodesStatus(); + assertEquals(1, nodesStatus.size()); + + response = CollectionAdminRequest.deleteCollection("solrj_implicit", server); + assertEquals(0, response.getStatus()); + assertTrue(response.isSuccess()); + nodesStatus = response.getCollectionNodesStatus(); + assertEquals(2, nodesStatus.size()); + + response = CollectionAdminRequest.createCollection("conf1", 4, server); + assertEquals(0, response.getStatus()); + assertTrue(response.isSuccess()); + + response = CollectionAdminRequest.reloadCollection("conf1", server); + assertEquals(0, response.getStatus()); + + response = CollectionAdminRequest.createAlias("solrj_alias", "conf1,solrj_collection", server); + assertEquals(0, response.getStatus()); + + response = CollectionAdminRequest.deleteAlias("solrj_alias", server); + assertEquals(0, response.getStatus()); + + response = CollectionAdminRequest.splitShard("conf1", "shard1", server); + assertEquals(0, response.getStatus()); + assertTrue(response.isSuccess()); + coresStatus = response.getCollectionCoresStatus(); + assertEquals(0, (int) coresStatus.get("conf1_shard1_0_replica1").get("status")); + assertEquals(0, (int) coresStatus.get("conf1_shard1_0_replica1").get("status")); + + response = CollectionAdminRequest.deleteCollection("conf1", server); + assertEquals(0, response.getStatus()); + nodesStatus = response.getCollectionNodesStatus(); + assertTrue(response.isSuccess()); + assertEquals(4, nodesStatus.size()); + + response = CollectionAdminRequest.deleteCollection("solrj_collection", server); + assertEquals(0, response.getStatus()); + nodesStatus = response.getCollectionNodesStatus(); + assertTrue(response.isSuccess()); + assertEquals(4, nodesStatus.size()); + } + + private void deletePartiallyCreatedCollection() throws Exception { final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0)); String collectionName = "halfdeletedcollection"; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java new file mode 100644 index 00000000000..502e4af0c77 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -0,0 +1,402 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.request; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ContentStream; + +/** + * This class is experimental and subject to change. + * + * @since solr 4.5 + */ +public class CollectionAdminRequest extends SolrRequest +{ + protected String collection = null; + protected CollectionAction action = null; + + protected static class CollectionShardAdminRequest extends CollectionAdminRequest { + protected String shardName = null; + + public void setShardName(String shard) { this.shardName = shard; } + public String getShardName() { return this.shardName; } + + public ModifiableSolrParams getCommonParams() { + ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + params.remove( "name" ); + params.set( "collection", collection ); + params.set( "shard", shardName); + return params; + } + + @Override + public SolrParams getParams() { + return getCommonParams(); + } + } + + //a create collection request + public static class Create extends CollectionAdminRequest { + protected String configName = null; + protected String createNodeSet = null; + protected String routerName; + protected String shards; + protected String routerField; + protected Integer numShards; + protected Integer maxShardsPerNode; + protected Integer replicationFactor; + + + public Create() { + action = CollectionAction.CREATE; + } + + public void setConfigName(String config) { this.configName = config; } + public void setCreateNodeSet(String nodeSet) { this.createNodeSet = nodeSet; } + public void setRouterName(String routerName) { this.routerName = routerName; } + public void setShards(String shards) { this.shards = shards; } + public void setRouterField(String routerField) { this.routerField = routerField; } + public void setNumShards(Integer numShards) {this.numShards = numShards;} + public void setMaxShardsPerNode(Integer numShards) { this.maxShardsPerNode = numShards; } + public void setReplicationFactor(Integer repl) { this.replicationFactor = repl; } + + public String getConfigName() { return configName; } + public String getCreateNodeSet() { return createNodeSet; } + public String getRouterName() { return routerName; } + public String getShards() { return shards; } + public Integer getNumShards() { return numShards; } + public Integer getMaxShardsPerNode() { return maxShardsPerNode; } + public Integer getReplicationFactor() { return replicationFactor; } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + + if (configName != null) { + params.set( "collection.configName", configName); + } + if (createNodeSet != null) { + params.set( "createNodeSet", createNodeSet); + } + if (numShards != null) { + params.set( ZkStateReader.NUM_SHARDS_PROP, numShards); + } + if (maxShardsPerNode != null) { + params.set( "maxShardsPerNode", maxShardsPerNode); + } + if (routerName != null) { + params.set( "router.name", routerName); + } + if (shards != null) { + params.set("shards", shards); + } + if (routerField != null) { + params.set("router.field", routerField); + } + if (replicationFactor != null) { + // OverseerCollectionProcessor.REPLICATION_FACTOR + params.set( "replicationFactor", replicationFactor); + } + + return params; + } + } + + //a reload collection request + public static class Reload extends CollectionAdminRequest { + public Reload() { + action = CollectionAction.RELOAD; + } + } + + //a delete collection request + public static class Delete extends CollectionAdminRequest { + public Delete() { + action = CollectionAction.DELETE; + } + } + + //a create shard collection request + public static class CreateShard extends CollectionShardAdminRequest { + protected String nodeSet; + + public void setNodeSet(String nodeSet) { this.nodeSet = nodeSet; } + public String getNodeSet() { return nodeSet; } + + public CreateShard() { + action = CollectionAction.CREATESHARD; + } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = getCommonParams(); + params.set( "createNodeSet", nodeSet); + return params; + } + } + + //a split shard collection request + public static class SplitShard extends CollectionShardAdminRequest { + protected String ranges; + + public void setRanges(String ranges) { this.ranges = ranges; } + public String getRanges() { return ranges; } + + public SplitShard() { + action = CollectionAction.SPLITSHARD; + } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = getCommonParams(); + params.set( "ranges", ranges); + return params; + } + } + + //a delete shard collection request + public static class DeleteShard extends CollectionShardAdminRequest { + public DeleteShard() { + action = CollectionAction.DELETESHARD; + } + } + + //a collection alias create request + public static class CreateAlias extends CollectionAdminRequest { + protected String aliasedCollections = null; + + public void setAliasedCollections(String alias) { this.aliasedCollections = alias; } + public String getAliasedCollections() { return this.aliasedCollections; } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + params.set( "collections", collection ); + return params; + } + + + public CreateAlias() { + action = CollectionAction.CREATEALIAS; + } + } + + //a collection alias delete request + public static class DeleteAlias extends CollectionAdminRequest { + public DeleteAlias() { + action = CollectionAction.DELETEALIAS; + } + } + + public CollectionAdminRequest() + { + super( METHOD.GET, "/admin/collections" ); + } + + public CollectionAdminRequest( String path ) + { + super( METHOD.GET, path ); + } + + public final void setCollectionName( String collectionName ) + { + this.collection = collectionName; + } + + //--------------------------------------------------------------------------------------- + // + //--------------------------------------------------------------------------------------- + + public void setAction( CollectionAction action ) + { + this.action = action; + } + + //--------------------------------------------------------------------------------------- + // + //--------------------------------------------------------------------------------------- + + @Override + public SolrParams getParams() + { + if( action == null ) { + throw new RuntimeException( "no action specified!" ); + } + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set( CoreAdminParams.ACTION, action.toString() ); + params.set( CoreAdminParams.NAME, collection ); + return params; + } + + //--------------------------------------------------------------------------------------- + // + //--------------------------------------------------------------------------------------- + + @Override + public Collection getContentStreams() throws IOException { + return null; + } + + @Override + public CollectionAdminResponse process(SolrServer server) throws SolrServerException, IOException + { + long startTime = System.currentTimeMillis(); + CollectionAdminResponse res = new CollectionAdminResponse(); + res.setResponse( server.request( this ) ); + res.setElapsedTime( System.currentTimeMillis()-startTime ); + return res; + } + + //--------------------------------------------------------------------------------------- + // + //--------------------------------------------------------------------------------------- + + // creates collection using a compositeId router + public static CollectionAdminResponse createCollection( String name, + Integer shards, Integer repl, Integer maxShards, + String nodeSet, + String conf, + String routerField, + SolrServer server ) throws SolrServerException, IOException + { + Create req = new Create(); + req.setCollectionName(name); + req.setRouterName("compositeId"); + req.setNumShards(shards); + req.setReplicationFactor(repl); + req.setMaxShardsPerNode(maxShards); + req.setCreateNodeSet(nodeSet); + req.setConfigName(conf); + req.setRouterField(routerField); + return req.process( server ); + } + public static CollectionAdminResponse createCollection( String name, + Integer shards, + SolrServer server ) throws SolrServerException, IOException + { + Create req = new Create(); + req.setCollectionName(name); + req.setRouterName("compositeId"); + req.setNumShards(shards); + return req.process( server ); + } + + // creates a collection using an implicit router + public static CollectionAdminResponse createCollection( String name, + String shards, Integer repl, Integer maxShards, + String nodeSet, + String conf, + String routerField, + SolrServer server ) throws SolrServerException, IOException + { + Create req = new Create(); + req.setCollectionName(name); + req.setRouterName("implicit"); + req.setShards(shards); + req.setReplicationFactor(repl); + req.setMaxShardsPerNode(maxShards); + req.setCreateNodeSet(nodeSet); + req.setConfigName(conf); + req.setRouterField(routerField); + return req.process( server ); + } + public static CollectionAdminResponse createCollection( String name, + String shards, + SolrServer server ) throws SolrServerException, IOException + { + Create req = new Create(); + req.setCollectionName(name); + req.setRouterName("implicit"); + req.setShards(shards); + return req.process( server ); + } + + public static CollectionAdminResponse reloadCollection( String name, SolrServer server ) throws SolrServerException, IOException + { + CollectionAdminRequest req = new Reload(); + req.setCollectionName(name); + return req.process( server ); + } + + public static CollectionAdminResponse deleteCollection( String name, SolrServer server ) throws SolrServerException, IOException + { + CollectionAdminRequest req = new Delete(); + req.setCollectionName(name); + return req.process( server ); + } + + public static CollectionAdminResponse createShard( String name, String shard, String nodeSet, SolrServer server ) throws SolrServerException, IOException + { + CreateShard req = new CreateShard(); + req.setCollectionName(name); + req.setShardName(shard); + req.setNodeSet(nodeSet); + return req.process( server ); + } + public static CollectionAdminResponse createShard( String name, String shard, SolrServer server ) throws SolrServerException, IOException + { + return createShard(name, shard, null, server); + } + + public static CollectionAdminResponse splitShard( String name, String shard, String ranges, SolrServer server ) throws SolrServerException, IOException + { + SplitShard req = new SplitShard(); + req.setCollectionName(name); + req.setShardName(shard); + req.setRanges(ranges); + return req.process( server ); + } + public static CollectionAdminResponse splitShard( String name, String shard, SolrServer server ) throws SolrServerException, IOException + { + return splitShard(name, shard, null, server); + } + + public static CollectionAdminResponse deleteShard( String name, String shard, SolrServer server ) throws SolrServerException, IOException + { + CollectionShardAdminRequest req = new DeleteShard(); + req.setCollectionName(name); + req.setShardName(shard); + return req.process( server ); + } + + public static CollectionAdminResponse createAlias( String name, String collections, SolrServer server ) throws SolrServerException, IOException + { + CreateAlias req = new CreateAlias(); + req.setCollectionName(name); + req.setAliasedCollections(collections); + return req.process( server ); + } + + public static CollectionAdminResponse deleteAlias( String name, SolrServer server ) throws SolrServerException, IOException + { + CollectionAdminRequest req = new DeleteAlias(); + req.setCollectionName(name); + return req.process( server ); + } +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java new file mode 100644 index 00000000000..3a6b643c77c --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.response; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.solr.common.util.NamedList; + +public class CollectionAdminResponse extends SolrResponseBase +{ + @SuppressWarnings("unchecked") + public NamedList> getCollectionStatus() + { + return (NamedList>) getResponse().get( "success" ); + } + + public boolean isSuccess() + { + return getResponse().get( "success" ) != null; + } + + // this messages are typically from individual nodes, since + // all the failures at the router are propagated as exceptions + @SuppressWarnings("unchecked") + public NamedList getErrorMessages() + { + return (NamedList) getResponse().get( "failure" ); + } + + @SuppressWarnings("unchecked") + public Map> getCollectionCoresStatus() + { + Map> res = new HashMap>(); + NamedList> cols = getCollectionStatus(); + if( cols != null ) { + for (Map.Entry> e : cols) { + NamedList item = e.getValue(); + String core = (String) item.get("core"); + if (core != null) { + res.put(core, (NamedList)item.get("responseHeader")); + } + } + } + + return res; + } + + @SuppressWarnings("unchecked") + public Map> getCollectionNodesStatus() + { + Map> res = new HashMap>(); + NamedList> cols = getCollectionStatus(); + if( cols != null ) { + for (Map.Entry> e : cols) { + if (e.getKey() != null) { + res.put(e.getKey(), (NamedList) (e.getValue().get("responseHeader"))); + } + } + } + + return res; + } +} \ No newline at end of file