Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-5300: Shards can be split by specifying arbitrary number of hash ranges within the shard's hash range 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1529444 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2013-10-05 13:00:36 +00:00
parent 2b1bea8279
commit 2cd31e54b5
7 changed files with 83 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
solr/CHANGES.txt
View File

@ -84,7 +84,10 @@ New Features
(shalin)
* SOLR-5274: Allow JettySolrRunner SSL config to be specified via a constructor.
(Mark Miller)
(Mark Miller)
* SOLR-5300: Shards can be split by specifying arbitrary number of hash ranges
within the shard's hash range. (shalin)
Bug Fixes
----------------------

30
solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
View File

@ -484,9 +484,32 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
range = new PlainIdRouter().fullRange();
}
// todo: fixed to two partitions?
// todo: accept the range as a param to api?
List<DocRouter.Range> subRanges = router.partitionRange(2, range);
List<DocRouter.Range> subRanges = null;
String rangesStr = message.getStr(CoreAdminParams.RANGES);
if (rangesStr != null) {
String[] ranges = rangesStr.split(",");
if (ranges.length == 0 || ranges.length == 1) {
throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split a shard");
} else {
subRanges = new ArrayList<DocRouter.Range>(ranges.length);
for (int i = 0; i < ranges.length; i++) {
String r = ranges[i];
try {
subRanges.add(DocRouter.DEFAULT.fromString(r));
} catch (Exception e) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Exception in parsing hexadecimal hash range: " + r, e);
}
if (!subRanges.get(i).isSubsetOf(range)) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Specified hash range: " + r + " is not a subset of parent shard's range: " + range.toString());
}
}
}
} else {
// todo: fixed to two partitions?
subRanges = router.partitionRange(2, range);
}
try {
List<String> subSlices = new ArrayList<String>(subRanges.size());
List<String> subShardNames = new ArrayList<String>(subRanges.size());
@ -579,6 +602,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
String subShardName = subShardNames.get(i);
params.add(CoreAdminParams.TARGET_CORE, subShardName);
}
params.set(CoreAdminParams.RANGES, rangesStr);
sendShardRequest(parentShardLeader.getNodeName(), params);
collectShardResponses(results, true, "SPLITSHARD failed to invoke SPLIT core admin command");

5
solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View File

@ -370,12 +370,15 @@ public class CollectionsHandler extends RequestHandlerBase {
String name = req.getParams().required().get("collection");
// TODO : add support for multiple shards
String shard = req.getParams().required().get("shard");
// TODO : add support for shard range
String rangesStr = req.getParams().get(CoreAdminParams.RANGES);
Map<String,Object> props = new HashMap<String,Object>();
props.put(Overseer.QUEUE_OPERATION, OverseerCollectionProcessor.SPLITSHARD);
props.put("collection", name);
props.put(ZkStateReader.SHARD_ID_PROP, shard);
if (rangesStr != null) {
props.put(CoreAdminParams.RANGES, rangesStr);
}
ZkNodeProps m = new ZkNodeProps(props);

23
solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
View File

@ -231,7 +231,22 @@ public class CoreAdminHandler extends RequestHandlerBase {
List<DocRouter.Range> ranges = null;
String[] pathsArr = params.getParams("path");
String rangesStr = params.get("ranges"); // ranges=a-b,c-d,e-f
String rangesStr = params.get(CoreAdminParams.RANGES); // ranges=a-b,c-d,e-f
if (rangesStr != null) {
String[] rangesArr = rangesStr.split(",");
if (rangesArr.length == 0 || rangesArr.length == 1) {
throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split an index");
} else {
ranges = new ArrayList<DocRouter.Range>(rangesArr.length);
for (String r : rangesArr) {
try {
ranges.add(DocRouter.DEFAULT.fromString(r));
} catch (Exception e) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Exception parsing hexadecimal hash range: " + r, e);
}
}
}
}
String[] newCoreNames = params.getParams("targetCore");
String cname = params.get(CoreAdminParams.CORE, "");
@ -257,9 +272,11 @@ public class CoreAdminHandler extends RequestHandlerBase {
DocCollection collection = clusterState.getCollection(collectionName);
String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
Slice slice = clusterState.getSlice(collectionName, sliceName);
DocRouter.Range currentRange = slice.getRange();
router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
if (ranges == null) {
DocRouter.Range currentRange = slice.getRange();
ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
}
Map m = (Map) collection.get(DOC_ROUTER);
if (m != null) {
routeFieldName = (String) m.get("field");

2
solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java
View File

@ -135,7 +135,7 @@ public class ChaosMonkeyShardSplitTest extends ShardSplitTest {
killerThread.start();
killCounter.incrementAndGet();
splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, null);
log.info("Layout after split: \n");
printLayout();

29
solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
View File

@ -18,7 +18,6 @@ package org.apache.solr.cloud;
*/
import org.apache.http.params.CoreConnectionPNames;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
@ -45,6 +44,7 @@ import org.junit.Before;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -123,7 +123,16 @@ public class ShardSplitTest extends BasicDistributedZkTest {
final DocRouter router = clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getRouter();
Slice shard1 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange();
final List<DocRouter.Range> ranges = router.partitionRange(2, shard1Range);
List<DocRouter.Range> subRanges = new ArrayList<DocRouter.Range>();
if (usually()) {
List<DocRouter.Range> ranges = router.partitionRange(4, shard1Range);
// 75% of range goes to shard1_0 and the rest to shard1_1
subRanges.add(new DocRouter.Range(ranges.get(0).min, ranges.get(2).max));
subRanges.add(ranges.get(3));
} else {
subRanges = router.partitionRange(2, shard1Range);
}
final List<DocRouter.Range> ranges = subRanges;
final int[] docCounts = new int[ranges.size()];
int numReplicas = shard1.getReplicas().size();
@ -167,7 +176,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {
try {
for (int i = 0; i < 3; i++) {
try {
splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges);
log.info("Layout after split: \n");
printLayout();
break;
@ -252,7 +261,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {
for (int i = 0; i < 3; i++) {
try {
splitShard(collectionName, SHARD1);
splitShard(collectionName, SHARD1, null);
break;
} catch (HttpSolrServer.RemoteSolrException e) {
if (e.code() != 500) {
@ -339,11 +348,21 @@ public class ShardSplitTest extends BasicDistributedZkTest {
}
}
protected void splitShard(String collection, String shardId) throws SolrServerException, IOException {
protected void splitShard(String collection, String shardId, List<DocRouter.Range> subRanges) throws SolrServerException, IOException {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.SPLITSHARD.toString());
params.set("collection", collection);
params.set("shard", shardId);
if (subRanges != null) {
StringBuilder ranges = new StringBuilder();
for (int i = 0; i < subRanges.size(); i++) {
DocRouter.Range subRange = subRanges.get(i);
ranges.append(subRange.toString());
if (i < subRanges.size() - 1)
ranges.append(",");
}
params.set("ranges", ranges.toString());
}
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");

3
solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
View File

@ -83,6 +83,9 @@ public abstract class CoreAdminParams
/** The target core to which a split index should be written to
* Multiple targetCores can be specified by multiple targetCore parameters */
public final static String TARGET_CORE = "targetCore";
/** The hash ranges to be used to split a shard or an index */
public final static String RANGES = "ranges";
public static final String ROLES = "roles";