mirror of https://github.com/apache/lucene.git
SOLR-5300: Shards can be split by specifying arbitrary number of hash ranges within the shard's hash range
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1529444 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2b1bea8279
commit
2cd31e54b5
|
@ -84,7 +84,10 @@ New Features
|
|||
(shalin)
|
||||
|
||||
* SOLR-5274: Allow JettySolrRunner SSL config to be specified via a constructor.
|
||||
(Mark Miller)
|
||||
(Mark Miller)
|
||||
|
||||
* SOLR-5300: Shards can be split by specifying arbitrary number of hash ranges
|
||||
within the shard's hash range. (shalin)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
|
|
@ -484,9 +484,32 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
|
|||
range = new PlainIdRouter().fullRange();
|
||||
}
|
||||
|
||||
// todo: fixed to two partitions?
|
||||
// todo: accept the range as a param to api?
|
||||
List<DocRouter.Range> subRanges = router.partitionRange(2, range);
|
||||
List<DocRouter.Range> subRanges = null;
|
||||
String rangesStr = message.getStr(CoreAdminParams.RANGES);
|
||||
if (rangesStr != null) {
|
||||
String[] ranges = rangesStr.split(",");
|
||||
if (ranges.length == 0 || ranges.length == 1) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split a shard");
|
||||
} else {
|
||||
subRanges = new ArrayList<DocRouter.Range>(ranges.length);
|
||||
for (int i = 0; i < ranges.length; i++) {
|
||||
String r = ranges[i];
|
||||
try {
|
||||
subRanges.add(DocRouter.DEFAULT.fromString(r));
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Exception in parsing hexadecimal hash range: " + r, e);
|
||||
}
|
||||
if (!subRanges.get(i).isSubsetOf(range)) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST,
|
||||
"Specified hash range: " + r + " is not a subset of parent shard's range: " + range.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// todo: fixed to two partitions?
|
||||
subRanges = router.partitionRange(2, range);
|
||||
}
|
||||
|
||||
try {
|
||||
List<String> subSlices = new ArrayList<String>(subRanges.size());
|
||||
List<String> subShardNames = new ArrayList<String>(subRanges.size());
|
||||
|
@ -579,6 +602,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
|
|||
String subShardName = subShardNames.get(i);
|
||||
params.add(CoreAdminParams.TARGET_CORE, subShardName);
|
||||
}
|
||||
params.set(CoreAdminParams.RANGES, rangesStr);
|
||||
|
||||
sendShardRequest(parentShardLeader.getNodeName(), params);
|
||||
collectShardResponses(results, true, "SPLITSHARD failed to invoke SPLIT core admin command");
|
||||
|
|
|
@ -370,12 +370,15 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
String name = req.getParams().required().get("collection");
|
||||
// TODO : add support for multiple shards
|
||||
String shard = req.getParams().required().get("shard");
|
||||
// TODO : add support for shard range
|
||||
String rangesStr = req.getParams().get(CoreAdminParams.RANGES);
|
||||
|
||||
Map<String,Object> props = new HashMap<String,Object>();
|
||||
props.put(Overseer.QUEUE_OPERATION, OverseerCollectionProcessor.SPLITSHARD);
|
||||
props.put("collection", name);
|
||||
props.put(ZkStateReader.SHARD_ID_PROP, shard);
|
||||
if (rangesStr != null) {
|
||||
props.put(CoreAdminParams.RANGES, rangesStr);
|
||||
}
|
||||
|
||||
ZkNodeProps m = new ZkNodeProps(props);
|
||||
|
||||
|
|
|
@ -231,7 +231,22 @@ public class CoreAdminHandler extends RequestHandlerBase {
|
|||
List<DocRouter.Range> ranges = null;
|
||||
|
||||
String[] pathsArr = params.getParams("path");
|
||||
String rangesStr = params.get("ranges"); // ranges=a-b,c-d,e-f
|
||||
String rangesStr = params.get(CoreAdminParams.RANGES); // ranges=a-b,c-d,e-f
|
||||
if (rangesStr != null) {
|
||||
String[] rangesArr = rangesStr.split(",");
|
||||
if (rangesArr.length == 0 || rangesArr.length == 1) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split an index");
|
||||
} else {
|
||||
ranges = new ArrayList<DocRouter.Range>(rangesArr.length);
|
||||
for (String r : rangesArr) {
|
||||
try {
|
||||
ranges.add(DocRouter.DEFAULT.fromString(r));
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(ErrorCode.BAD_REQUEST, "Exception parsing hexadecimal hash range: " + r, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
String[] newCoreNames = params.getParams("targetCore");
|
||||
String cname = params.get(CoreAdminParams.CORE, "");
|
||||
|
||||
|
@ -257,9 +272,11 @@ public class CoreAdminHandler extends RequestHandlerBase {
|
|||
DocCollection collection = clusterState.getCollection(collectionName);
|
||||
String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
|
||||
Slice slice = clusterState.getSlice(collectionName, sliceName);
|
||||
DocRouter.Range currentRange = slice.getRange();
|
||||
router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
|
||||
ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
|
||||
if (ranges == null) {
|
||||
DocRouter.Range currentRange = slice.getRange();
|
||||
ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
|
||||
}
|
||||
Map m = (Map) collection.get(DOC_ROUTER);
|
||||
if (m != null) {
|
||||
routeFieldName = (String) m.get("field");
|
||||
|
|
|
@ -135,7 +135,7 @@ public class ChaosMonkeyShardSplitTest extends ShardSplitTest {
|
|||
killerThread.start();
|
||||
killCounter.incrementAndGet();
|
||||
|
||||
splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
|
||||
splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, null);
|
||||
|
||||
log.info("Layout after split: \n");
|
||||
printLayout();
|
||||
|
|
|
@ -18,7 +18,6 @@ package org.apache.solr.cloud;
|
|||
*/
|
||||
|
||||
import org.apache.http.params.CoreConnectionPNames;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.solr.client.solrj.SolrQuery;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
|
@ -45,6 +44,7 @@ import org.junit.Before;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
@ -123,7 +123,16 @@ public class ShardSplitTest extends BasicDistributedZkTest {
|
|||
final DocRouter router = clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getRouter();
|
||||
Slice shard1 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
|
||||
DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange();
|
||||
final List<DocRouter.Range> ranges = router.partitionRange(2, shard1Range);
|
||||
List<DocRouter.Range> subRanges = new ArrayList<DocRouter.Range>();
|
||||
if (usually()) {
|
||||
List<DocRouter.Range> ranges = router.partitionRange(4, shard1Range);
|
||||
// 75% of range goes to shard1_0 and the rest to shard1_1
|
||||
subRanges.add(new DocRouter.Range(ranges.get(0).min, ranges.get(2).max));
|
||||
subRanges.add(ranges.get(3));
|
||||
} else {
|
||||
subRanges = router.partitionRange(2, shard1Range);
|
||||
}
|
||||
final List<DocRouter.Range> ranges = subRanges;
|
||||
final int[] docCounts = new int[ranges.size()];
|
||||
int numReplicas = shard1.getReplicas().size();
|
||||
|
||||
|
@ -167,7 +176,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {
|
|||
try {
|
||||
for (int i = 0; i < 3; i++) {
|
||||
try {
|
||||
splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
|
||||
splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges);
|
||||
log.info("Layout after split: \n");
|
||||
printLayout();
|
||||
break;
|
||||
|
@ -252,7 +261,7 @@ public class ShardSplitTest extends BasicDistributedZkTest {
|
|||
|
||||
for (int i = 0; i < 3; i++) {
|
||||
try {
|
||||
splitShard(collectionName, SHARD1);
|
||||
splitShard(collectionName, SHARD1, null);
|
||||
break;
|
||||
} catch (HttpSolrServer.RemoteSolrException e) {
|
||||
if (e.code() != 500) {
|
||||
|
@ -339,11 +348,21 @@ public class ShardSplitTest extends BasicDistributedZkTest {
|
|||
}
|
||||
}
|
||||
|
||||
protected void splitShard(String collection, String shardId) throws SolrServerException, IOException {
|
||||
protected void splitShard(String collection, String shardId, List<DocRouter.Range> subRanges) throws SolrServerException, IOException {
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.set("action", CollectionParams.CollectionAction.SPLITSHARD.toString());
|
||||
params.set("collection", collection);
|
||||
params.set("shard", shardId);
|
||||
if (subRanges != null) {
|
||||
StringBuilder ranges = new StringBuilder();
|
||||
for (int i = 0; i < subRanges.size(); i++) {
|
||||
DocRouter.Range subRange = subRanges.get(i);
|
||||
ranges.append(subRange.toString());
|
||||
if (i < subRanges.size() - 1)
|
||||
ranges.append(",");
|
||||
}
|
||||
params.set("ranges", ranges.toString());
|
||||
}
|
||||
SolrRequest request = new QueryRequest(params);
|
||||
request.setPath("/admin/collections");
|
||||
|
||||
|
|
|
@ -83,6 +83,9 @@ public abstract class CoreAdminParams
|
|||
/** The target core to which a split index should be written to
|
||||
* Multiple targetCores can be specified by multiple targetCore parameters */
|
||||
public final static String TARGET_CORE = "targetCore";
|
||||
|
||||
/** The hash ranges to be used to split a shard or an index */
|
||||
public final static String RANGES = "ranges";
|
||||
|
||||
public static final String ROLES = "roles";
|
||||
|
||||
|
|
Loading…
Reference in New Issue