From 2cd31e54b54456753ae525c5247f90e347afe95b Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Sat, 5 Oct 2013 13:00:36 +0000 Subject: [PATCH] SOLR-5300: Shards can be split by specifying arbitrary number of hash ranges within the shard's hash range git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1529444 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 5 +++- .../cloud/OverseerCollectionProcessor.java | 30 +++++++++++++++++-- .../handler/admin/CollectionsHandler.java | 5 +++- .../solr/handler/admin/CoreAdminHandler.java | 23 ++++++++++++-- .../solr/cloud/ChaosMonkeyShardSplitTest.java | 2 +- .../org/apache/solr/cloud/ShardSplitTest.java | 29 ++++++++++++++---- .../solr/common/params/CoreAdminParams.java | 3 ++ 7 files changed, 83 insertions(+), 14 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 658fb9a55a7..a3db4045cef 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -84,7 +84,10 @@ New Features (shalin) * SOLR-5274: Allow JettySolrRunner SSL config to be specified via a constructor. - (Mark Miller) + (Mark Miller) + +* SOLR-5300: Shards can be split by specifying arbitrary number of hash ranges + within the shard's hash range. (shalin) Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java index 43355c043d3..7c173f0cefe 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java @@ -484,9 +484,32 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { range = new PlainIdRouter().fullRange(); } - // todo: fixed to two partitions? - // todo: accept the range as a param to api? - List subRanges = router.partitionRange(2, range); + List subRanges = null; + String rangesStr = message.getStr(CoreAdminParams.RANGES); + if (rangesStr != null) { + String[] ranges = rangesStr.split(","); + if (ranges.length == 0 || ranges.length == 1) { + throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split a shard"); + } else { + subRanges = new ArrayList(ranges.length); + for (int i = 0; i < ranges.length; i++) { + String r = ranges[i]; + try { + subRanges.add(DocRouter.DEFAULT.fromString(r)); + } catch (Exception e) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Exception in parsing hexadecimal hash range: " + r, e); + } + if (!subRanges.get(i).isSubsetOf(range)) { + throw new SolrException(ErrorCode.BAD_REQUEST, + "Specified hash range: " + r + " is not a subset of parent shard's range: " + range.toString()); + } + } + } + } else { + // todo: fixed to two partitions? + subRanges = router.partitionRange(2, range); + } + try { List subSlices = new ArrayList(subRanges.size()); List subShardNames = new ArrayList(subRanges.size()); @@ -579,6 +602,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { String subShardName = subShardNames.get(i); params.add(CoreAdminParams.TARGET_CORE, subShardName); } + params.set(CoreAdminParams.RANGES, rangesStr); sendShardRequest(parentShardLeader.getNodeName(), params); collectShardResponses(results, true, "SPLITSHARD failed to invoke SPLIT core admin command"); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 5e51ae6c567..0214259eacb 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -370,12 +370,15 @@ public class CollectionsHandler extends RequestHandlerBase { String name = req.getParams().required().get("collection"); // TODO : add support for multiple shards String shard = req.getParams().required().get("shard"); - // TODO : add support for shard range + String rangesStr = req.getParams().get(CoreAdminParams.RANGES); Map props = new HashMap(); props.put(Overseer.QUEUE_OPERATION, OverseerCollectionProcessor.SPLITSHARD); props.put("collection", name); props.put(ZkStateReader.SHARD_ID_PROP, shard); + if (rangesStr != null) { + props.put(CoreAdminParams.RANGES, rangesStr); + } ZkNodeProps m = new ZkNodeProps(props); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index 390d8ed749f..3ae4ada4b19 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -231,7 +231,22 @@ public class CoreAdminHandler extends RequestHandlerBase { List ranges = null; String[] pathsArr = params.getParams("path"); - String rangesStr = params.get("ranges"); // ranges=a-b,c-d,e-f + String rangesStr = params.get(CoreAdminParams.RANGES); // ranges=a-b,c-d,e-f + if (rangesStr != null) { + String[] rangesArr = rangesStr.split(","); + if (rangesArr.length == 0 || rangesArr.length == 1) { + throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split an index"); + } else { + ranges = new ArrayList(rangesArr.length); + for (String r : rangesArr) { + try { + ranges.add(DocRouter.DEFAULT.fromString(r)); + } catch (Exception e) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Exception parsing hexadecimal hash range: " + r, e); + } + } + } + } String[] newCoreNames = params.getParams("targetCore"); String cname = params.get(CoreAdminParams.CORE, ""); @@ -257,9 +272,11 @@ public class CoreAdminHandler extends RequestHandlerBase { DocCollection collection = clusterState.getCollection(collectionName); String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId(); Slice slice = clusterState.getSlice(collectionName, sliceName); - DocRouter.Range currentRange = slice.getRange(); router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT; - ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null; + if (ranges == null) { + DocRouter.Range currentRange = slice.getRange(); + ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null; + } Map m = (Map) collection.get(DOC_ROUTER); if (m != null) { routeFieldName = (String) m.get("field"); diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java index 1111f2d9954..51198088523 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java @@ -135,7 +135,7 @@ public class ChaosMonkeyShardSplitTest extends ShardSplitTest { killerThread.start(); killCounter.incrementAndGet(); - splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1); + splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, null); log.info("Layout after split: \n"); printLayout(); diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java index c339f7b532d..c69b02d2b41 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java @@ -18,7 +18,6 @@ package org.apache.solr.cloud; */ import org.apache.http.params.CoreConnectionPNames; -import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServer; @@ -45,6 +44,7 @@ import org.junit.Before; import java.io.IOException; import java.net.MalformedURLException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -123,7 +123,16 @@ public class ShardSplitTest extends BasicDistributedZkTest { final DocRouter router = clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getRouter(); Slice shard1 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1); DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange(); - final List ranges = router.partitionRange(2, shard1Range); + List subRanges = new ArrayList(); + if (usually()) { + List ranges = router.partitionRange(4, shard1Range); + // 75% of range goes to shard1_0 and the rest to shard1_1 + subRanges.add(new DocRouter.Range(ranges.get(0).min, ranges.get(2).max)); + subRanges.add(ranges.get(3)); + } else { + subRanges = router.partitionRange(2, shard1Range); + } + final List ranges = subRanges; final int[] docCounts = new int[ranges.size()]; int numReplicas = shard1.getReplicas().size(); @@ -167,7 +176,7 @@ public class ShardSplitTest extends BasicDistributedZkTest { try { for (int i = 0; i < 3; i++) { try { - splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1); + splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges); log.info("Layout after split: \n"); printLayout(); break; @@ -252,7 +261,7 @@ public class ShardSplitTest extends BasicDistributedZkTest { for (int i = 0; i < 3; i++) { try { - splitShard(collectionName, SHARD1); + splitShard(collectionName, SHARD1, null); break; } catch (HttpSolrServer.RemoteSolrException e) { if (e.code() != 500) { @@ -339,11 +348,21 @@ public class ShardSplitTest extends BasicDistributedZkTest { } } - protected void splitShard(String collection, String shardId) throws SolrServerException, IOException { + protected void splitShard(String collection, String shardId, List subRanges) throws SolrServerException, IOException { ModifiableSolrParams params = new ModifiableSolrParams(); params.set("action", CollectionParams.CollectionAction.SPLITSHARD.toString()); params.set("collection", collection); params.set("shard", shardId); + if (subRanges != null) { + StringBuilder ranges = new StringBuilder(); + for (int i = 0; i < subRanges.size(); i++) { + DocRouter.Range subRange = subRanges.get(i); + ranges.append(subRange.toString()); + if (i < subRanges.size() - 1) + ranges.append(","); + } + params.set("ranges", ranges.toString()); + } SolrRequest request = new QueryRequest(params); request.setPath("/admin/collections"); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java index f9eefbc4bfe..d1ec6865e7e 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java @@ -83,6 +83,9 @@ public abstract class CoreAdminParams /** The target core to which a split index should be written to * Multiple targetCores can be specified by multiple targetCore parameters */ public final static String TARGET_CORE = "targetCore"; + + /** The hash ranges to be used to split a shard or an index */ + public final static String RANGES = "ranges"; public static final String ROLES = "roles";