SOLR-5392: Extend solrj apis to cover collection management.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1537787 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-11-01 00:50:08 +00:00
parent b449d3df60
commit 5952539b07
4 changed files with 567 additions and 1 deletions

View File

@ -119,7 +119,8 @@ New Features
via the optional DocBasedVersionConstraintsProcessorFactory
update processor (Hossman, yonik)
* SOLR-5392: Extend solrj apis to cover collection management.
(Roman Shaposhnik via Mark Miller)
Bug Fixes
----------------------

View File

@ -48,14 +48,17 @@ import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util._TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
@ -189,6 +192,7 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
@Override
public void doTest() throws Exception {
testSolrJAPICalls();
testNodesUsedByCreate();
testCollectionsAPI();
testErrorHandling();
@ -235,6 +239,86 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
}
private void testSolrJAPICalls() throws Exception {
SolrServer server = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0)));
CollectionAdminResponse response;
Map<String, NamedList<Integer>> coresStatus;
Map<String, NamedList<Integer>> nodesStatus;
response = CollectionAdminRequest.createCollection("solrj_collection",
2, 2, null,
null, "conf1", "myOwnField",
server);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
coresStatus = response.getCollectionCoresStatus();
assertEquals(4, coresStatus.size());
for (int i=0; i<4; i++) {
NamedList<Integer> status = coresStatus.get("solrj_collection_shard" + (i/2+1) + "_replica" + (i%2+1));
assertEquals(0, (int)status.get("status"));
assertTrue(status.get("QTime") > 0);
}
response = CollectionAdminRequest.createCollection("solrj_implicit",
"shardA,shardB", server);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
coresStatus = response.getCollectionCoresStatus();
assertEquals(2, coresStatus.size());
response = CollectionAdminRequest.createShard("solrj_implicit", "shardC", server);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
coresStatus = response.getCollectionCoresStatus();
assertEquals(1, coresStatus.size());
assertEquals(0, (int) coresStatus.get("solrj_implicit_shardC_replica1").get("status"));
response = CollectionAdminRequest.deleteShard("solrj_implicit", "shardC", server);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
nodesStatus = response.getCollectionNodesStatus();
assertEquals(1, nodesStatus.size());
response = CollectionAdminRequest.deleteCollection("solrj_implicit", server);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
nodesStatus = response.getCollectionNodesStatus();
assertEquals(2, nodesStatus.size());
response = CollectionAdminRequest.createCollection("conf1", 4, server);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
response = CollectionAdminRequest.reloadCollection("conf1", server);
assertEquals(0, response.getStatus());
response = CollectionAdminRequest.createAlias("solrj_alias", "conf1,solrj_collection", server);
assertEquals(0, response.getStatus());
response = CollectionAdminRequest.deleteAlias("solrj_alias", server);
assertEquals(0, response.getStatus());
response = CollectionAdminRequest.splitShard("conf1", "shard1", server);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
coresStatus = response.getCollectionCoresStatus();
assertEquals(0, (int) coresStatus.get("conf1_shard1_0_replica1").get("status"));
assertEquals(0, (int) coresStatus.get("conf1_shard1_0_replica1").get("status"));
response = CollectionAdminRequest.deleteCollection("conf1", server);
assertEquals(0, response.getStatus());
nodesStatus = response.getCollectionNodesStatus();
assertTrue(response.isSuccess());
assertEquals(4, nodesStatus.size());
response = CollectionAdminRequest.deleteCollection("solrj_collection", server);
assertEquals(0, response.getStatus());
nodesStatus = response.getCollectionNodesStatus();
assertTrue(response.isSuccess());
assertEquals(4, nodesStatus.size());
}
private void deletePartiallyCreatedCollection() throws Exception {
final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
String collectionName = "halfdeletedcollection";

View File

@ -0,0 +1,402 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.request;
import java.io.IOException;
import java.util.Collection;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
/**
* This class is experimental and subject to change.
*
* @since solr 4.5
*/
public class CollectionAdminRequest extends SolrRequest
{
protected String collection = null;
protected CollectionAction action = null;
protected static class CollectionShardAdminRequest extends CollectionAdminRequest {
protected String shardName = null;
public void setShardName(String shard) { this.shardName = shard; }
public String getShardName() { return this.shardName; }
public ModifiableSolrParams getCommonParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.remove( "name" );
params.set( "collection", collection );
params.set( "shard", shardName);
return params;
}
@Override
public SolrParams getParams() {
return getCommonParams();
}
}
//a create collection request
public static class Create extends CollectionAdminRequest {
protected String configName = null;
protected String createNodeSet = null;
protected String routerName;
protected String shards;
protected String routerField;
protected Integer numShards;
protected Integer maxShardsPerNode;
protected Integer replicationFactor;
public Create() {
action = CollectionAction.CREATE;
}
public void setConfigName(String config) { this.configName = config; }
public void setCreateNodeSet(String nodeSet) { this.createNodeSet = nodeSet; }
public void setRouterName(String routerName) { this.routerName = routerName; }
public void setShards(String shards) { this.shards = shards; }
public void setRouterField(String routerField) { this.routerField = routerField; }
public void setNumShards(Integer numShards) {this.numShards = numShards;}
public void setMaxShardsPerNode(Integer numShards) { this.maxShardsPerNode = numShards; }
public void setReplicationFactor(Integer repl) { this.replicationFactor = repl; }
public String getConfigName() { return configName; }
public String getCreateNodeSet() { return createNodeSet; }
public String getRouterName() { return routerName; }
public String getShards() { return shards; }
public Integer getNumShards() { return numShards; }
public Integer getMaxShardsPerNode() { return maxShardsPerNode; }
public Integer getReplicationFactor() { return replicationFactor; }
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
if (configName != null) {
params.set( "collection.configName", configName);
}
if (createNodeSet != null) {
params.set( "createNodeSet", createNodeSet);
}
if (numShards != null) {
params.set( ZkStateReader.NUM_SHARDS_PROP, numShards);
}
if (maxShardsPerNode != null) {
params.set( "maxShardsPerNode", maxShardsPerNode);
}
if (routerName != null) {
params.set( "router.name", routerName);
}
if (shards != null) {
params.set("shards", shards);
}
if (routerField != null) {
params.set("router.field", routerField);
}
if (replicationFactor != null) {
// OverseerCollectionProcessor.REPLICATION_FACTOR
params.set( "replicationFactor", replicationFactor);
}
return params;
}
}
//a reload collection request
public static class Reload extends CollectionAdminRequest {
public Reload() {
action = CollectionAction.RELOAD;
}
}
//a delete collection request
public static class Delete extends CollectionAdminRequest {
public Delete() {
action = CollectionAction.DELETE;
}
}
//a create shard collection request
public static class CreateShard extends CollectionShardAdminRequest {
protected String nodeSet;
public void setNodeSet(String nodeSet) { this.nodeSet = nodeSet; }
public String getNodeSet() { return nodeSet; }
public CreateShard() {
action = CollectionAction.CREATESHARD;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = getCommonParams();
params.set( "createNodeSet", nodeSet);
return params;
}
}
//a split shard collection request
public static class SplitShard extends CollectionShardAdminRequest {
protected String ranges;
public void setRanges(String ranges) { this.ranges = ranges; }
public String getRanges() { return ranges; }
public SplitShard() {
action = CollectionAction.SPLITSHARD;
}
@Override
public SolrParams getParams() {
ModifiableSolrParams params = getCommonParams();
params.set( "ranges", ranges);
return params;
}
}
//a delete shard collection request
public static class DeleteShard extends CollectionShardAdminRequest {
public DeleteShard() {
action = CollectionAction.DELETESHARD;
}
}
//a collection alias create request
public static class CreateAlias extends CollectionAdminRequest {
protected String aliasedCollections = null;
public void setAliasedCollections(String alias) { this.aliasedCollections = alias; }
public String getAliasedCollections() { return this.aliasedCollections; }
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.set( "collections", collection );
return params;
}
public CreateAlias() {
action = CollectionAction.CREATEALIAS;
}
}
//a collection alias delete request
public static class DeleteAlias extends CollectionAdminRequest {
public DeleteAlias() {
action = CollectionAction.DELETEALIAS;
}
}
public CollectionAdminRequest()
{
super( METHOD.GET, "/admin/collections" );
}
public CollectionAdminRequest( String path )
{
super( METHOD.GET, path );
}
public final void setCollectionName( String collectionName )
{
this.collection = collectionName;
}
//---------------------------------------------------------------------------------------
//
//---------------------------------------------------------------------------------------
public void setAction( CollectionAction action )
{
this.action = action;
}
//---------------------------------------------------------------------------------------
//
//---------------------------------------------------------------------------------------
@Override
public SolrParams getParams()
{
if( action == null ) {
throw new RuntimeException( "no action specified!" );
}
ModifiableSolrParams params = new ModifiableSolrParams();
params.set( CoreAdminParams.ACTION, action.toString() );
params.set( CoreAdminParams.NAME, collection );
return params;
}
//---------------------------------------------------------------------------------------
//
//---------------------------------------------------------------------------------------
@Override
public Collection<ContentStream> getContentStreams() throws IOException {
return null;
}
@Override
public CollectionAdminResponse process(SolrServer server) throws SolrServerException, IOException
{
long startTime = System.currentTimeMillis();
CollectionAdminResponse res = new CollectionAdminResponse();
res.setResponse( server.request( this ) );
res.setElapsedTime( System.currentTimeMillis()-startTime );
return res;
}
//---------------------------------------------------------------------------------------
//
//---------------------------------------------------------------------------------------
// creates collection using a compositeId router
public static CollectionAdminResponse createCollection( String name,
Integer shards, Integer repl, Integer maxShards,
String nodeSet,
String conf,
String routerField,
SolrServer server ) throws SolrServerException, IOException
{
Create req = new Create();
req.setCollectionName(name);
req.setRouterName("compositeId");
req.setNumShards(shards);
req.setReplicationFactor(repl);
req.setMaxShardsPerNode(maxShards);
req.setCreateNodeSet(nodeSet);
req.setConfigName(conf);
req.setRouterField(routerField);
return req.process( server );
}
public static CollectionAdminResponse createCollection( String name,
Integer shards,
SolrServer server ) throws SolrServerException, IOException
{
Create req = new Create();
req.setCollectionName(name);
req.setRouterName("compositeId");
req.setNumShards(shards);
return req.process( server );
}
// creates a collection using an implicit router
public static CollectionAdminResponse createCollection( String name,
String shards, Integer repl, Integer maxShards,
String nodeSet,
String conf,
String routerField,
SolrServer server ) throws SolrServerException, IOException
{
Create req = new Create();
req.setCollectionName(name);
req.setRouterName("implicit");
req.setShards(shards);
req.setReplicationFactor(repl);
req.setMaxShardsPerNode(maxShards);
req.setCreateNodeSet(nodeSet);
req.setConfigName(conf);
req.setRouterField(routerField);
return req.process( server );
}
public static CollectionAdminResponse createCollection( String name,
String shards,
SolrServer server ) throws SolrServerException, IOException
{
Create req = new Create();
req.setCollectionName(name);
req.setRouterName("implicit");
req.setShards(shards);
return req.process( server );
}
public static CollectionAdminResponse reloadCollection( String name, SolrServer server ) throws SolrServerException, IOException
{
CollectionAdminRequest req = new Reload();
req.setCollectionName(name);
return req.process( server );
}
public static CollectionAdminResponse deleteCollection( String name, SolrServer server ) throws SolrServerException, IOException
{
CollectionAdminRequest req = new Delete();
req.setCollectionName(name);
return req.process( server );
}
public static CollectionAdminResponse createShard( String name, String shard, String nodeSet, SolrServer server ) throws SolrServerException, IOException
{
CreateShard req = new CreateShard();
req.setCollectionName(name);
req.setShardName(shard);
req.setNodeSet(nodeSet);
return req.process( server );
}
public static CollectionAdminResponse createShard( String name, String shard, SolrServer server ) throws SolrServerException, IOException
{
return createShard(name, shard, null, server);
}
public static CollectionAdminResponse splitShard( String name, String shard, String ranges, SolrServer server ) throws SolrServerException, IOException
{
SplitShard req = new SplitShard();
req.setCollectionName(name);
req.setShardName(shard);
req.setRanges(ranges);
return req.process( server );
}
public static CollectionAdminResponse splitShard( String name, String shard, SolrServer server ) throws SolrServerException, IOException
{
return splitShard(name, shard, null, server);
}
public static CollectionAdminResponse deleteShard( String name, String shard, SolrServer server ) throws SolrServerException, IOException
{
CollectionShardAdminRequest req = new DeleteShard();
req.setCollectionName(name);
req.setShardName(shard);
return req.process( server );
}
public static CollectionAdminResponse createAlias( String name, String collections, SolrServer server ) throws SolrServerException, IOException
{
CreateAlias req = new CreateAlias();
req.setCollectionName(name);
req.setAliasedCollections(collections);
return req.process( server );
}
public static CollectionAdminResponse deleteAlias( String name, SolrServer server ) throws SolrServerException, IOException
{
CollectionAdminRequest req = new DeleteAlias();
req.setCollectionName(name);
return req.process( server );
}
}

View File

@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.response;
import java.util.HashMap;
import java.util.Map;
import org.apache.solr.common.util.NamedList;
public class CollectionAdminResponse extends SolrResponseBase
{
@SuppressWarnings("unchecked")
public NamedList<NamedList<Object>> getCollectionStatus()
{
return (NamedList<NamedList<Object>>) getResponse().get( "success" );
}
public boolean isSuccess()
{
return getResponse().get( "success" ) != null;
}
// this messages are typically from individual nodes, since
// all the failures at the router are propagated as exceptions
@SuppressWarnings("unchecked")
public NamedList<String> getErrorMessages()
{
return (NamedList<String>) getResponse().get( "failure" );
}
@SuppressWarnings("unchecked")
public Map<String, NamedList<Integer>> getCollectionCoresStatus()
{
Map<String, NamedList<Integer>> res = new HashMap<String, NamedList<Integer>>();
NamedList<NamedList<Object>> cols = getCollectionStatus();
if( cols != null ) {
for (Map.Entry<String, NamedList<Object>> e : cols) {
NamedList<Object> item = e.getValue();
String core = (String) item.get("core");
if (core != null) {
res.put(core, (NamedList<Integer>)item.get("responseHeader"));
}
}
}
return res;
}
@SuppressWarnings("unchecked")
public Map<String, NamedList<Integer>> getCollectionNodesStatus()
{
Map<String, NamedList<Integer>> res = new HashMap<String, NamedList<Integer>>();
NamedList<NamedList<Object>> cols = getCollectionStatus();
if( cols != null ) {
for (Map.Entry<String,NamedList<Object>> e : cols) {
if (e.getKey() != null) {
res.put(e.getKey(), (NamedList<Integer>) (e.getValue().get("responseHeader")));
}
}
}
return res;
}
}