mirror of https://github.com/apache/lucene.git
SOLR-4120: Collection API: Support for specifying a list of Solr addresses to spread a new collection across.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1421999 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller
parent
3ab1790be7
commit
17569aa775
|
|
@ -151,6 +151,9 @@ New Features
|
|||
* SOLR-788: Distributed search support for MLT.
|
||||
(Matthew Woytowitz, Mike Anderson, Jamie Johnson, Mark Miller)
|
||||
|
||||
* SOLR-4120: Collection API: Support for specifying a list of Solr addresses to
|
||||
spread a new collection across. (Per Steffensen via Mark Miller)
|
||||
|
||||
Optimizations
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import org.apache.solr.common.cloud.ZooKeeperException;
|
|||
import org.apache.solr.common.params.CoreAdminParams;
|
||||
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.handler.component.ShardHandler;
|
||||
import org.apache.solr.handler.component.ShardRequest;
|
||||
import org.apache.solr.handler.component.ShardResponse;
|
||||
|
|
@ -50,6 +51,8 @@ public class OverseerCollectionProcessor implements Runnable {
|
|||
|
||||
public static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
|
||||
|
||||
public static final String CREATE_NODE_SET = "createNodeSet";
|
||||
|
||||
public static final String DELETECOLLECTION = "deletecollection";
|
||||
|
||||
public static final String CREATECOLLECTION = "createcollection";
|
||||
|
|
@ -150,7 +153,7 @@ public class OverseerCollectionProcessor implements Runnable {
|
|||
return false;
|
||||
}
|
||||
|
||||
private boolean processMessage(ZkNodeProps message, String operation) {
|
||||
protected boolean processMessage(ZkNodeProps message, String operation) {
|
||||
if (CREATECOLLECTION.equals(operation)) {
|
||||
return createCollection(zkStateReader.getClusterState(), message);
|
||||
} else if (DELETECOLLECTION.equals(operation)) {
|
||||
|
|
@ -181,6 +184,8 @@ public class OverseerCollectionProcessor implements Runnable {
|
|||
int repFactor = msgStrToInt(message, REPLICATION_FACTOR, 1);
|
||||
int numSlices = msgStrToInt(message, NUM_SLICES, 0);
|
||||
int maxShardsPerNode = msgStrToInt(message, MAX_SHARDS_PER_NODE, 1);
|
||||
String createNodeSetStr;
|
||||
List<String> createNodeList = ((createNodeSetStr = message.getStr(CREATE_NODE_SET)) == null)?null:StrUtils.splitSmart(createNodeSetStr, ",", true);
|
||||
|
||||
if (repFactor <= 0) {
|
||||
SolrException.log(log, REPLICATION_FACTOR + " must be > 0");
|
||||
|
|
@ -204,11 +209,12 @@ public class OverseerCollectionProcessor implements Runnable {
|
|||
Set<String> nodes = clusterState.getLiveNodes();
|
||||
List<String> nodeList = new ArrayList<String>(nodes.size());
|
||||
nodeList.addAll(nodes);
|
||||
if (createNodeList != null) nodeList.retainAll(createNodeList);
|
||||
Collections.shuffle(nodeList);
|
||||
|
||||
if (nodeList.size() <= 0) {
|
||||
log.error("Cannot create collection " + collectionName
|
||||
+ ". No live Solr-instaces");
|
||||
+ ". No live Solr-instaces" + ((createNodeList != null)?" among Solr-instances specified in " + CREATE_NODE_SET:""));
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -184,6 +184,7 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
String configName = req.getParams().get("collection.configName");
|
||||
String numShards = req.getParams().get(OverseerCollectionProcessor.NUM_SLICES);
|
||||
String maxShardsPerNode = req.getParams().get(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE);
|
||||
String createNodeSetStr = req.getParams().get(OverseerCollectionProcessor.CREATE_NODE_SET);
|
||||
|
||||
if (name == null) {
|
||||
log.error("Collection name is required to create a new collection");
|
||||
|
|
@ -201,6 +202,7 @@ public class CollectionsHandler extends RequestHandlerBase {
|
|||
}
|
||||
props.put(OverseerCollectionProcessor.NUM_SLICES, numShards);
|
||||
props.put(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
|
||||
props.put(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
|
||||
|
||||
ZkNodeProps m = new ZkNodeProps(props);
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ import org.apache.solr.common.params.CommonParams;
|
|||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.UpdateParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.core.SolrInfoMBean.Category;
|
||||
import org.apache.solr.update.DirectUpdateHandler2;
|
||||
|
|
@ -775,7 +776,7 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
|
|||
}
|
||||
|
||||
createCollection(collectionInfos, "awholynewcollection_" + i,
|
||||
numShards, replicationFactor, maxShardsPerNode, client);
|
||||
numShards, replicationFactor, maxShardsPerNode, client, null);
|
||||
} finally {
|
||||
if (client != null) client.shutdown();
|
||||
}
|
||||
|
|
@ -785,7 +786,7 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
|
|||
for (Entry<String,List<Integer>> entry : collectionInfosEntrySet) {
|
||||
String collection = entry.getKey();
|
||||
List<Integer> list = entry.getValue();
|
||||
checkForCollection(collection, list);
|
||||
checkForCollection(collection, list, null);
|
||||
|
||||
String url = getUrlFromZk(collection);
|
||||
|
||||
|
|
@ -892,7 +893,7 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
|
|||
List<Integer> list = new ArrayList<Integer> (2);
|
||||
list.add(1);
|
||||
list.add(2);
|
||||
checkForCollection(collectionName, list);
|
||||
checkForCollection(collectionName, list, null);
|
||||
|
||||
url = getUrlFromZk(collectionName);
|
||||
|
||||
|
|
@ -906,14 +907,14 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
|
|||
}
|
||||
|
||||
// test maxShardsPerNode
|
||||
int liveNodes = getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size();
|
||||
int numShards = (liveNodes/2) + 1;
|
||||
int numLiveNodes = getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size();
|
||||
int numShards = (numLiveNodes/2) + 1;
|
||||
int replicationFactor = 2;
|
||||
int maxShardsPerNode = 1;
|
||||
collectionInfos = new HashMap<String,List<Integer>>();
|
||||
CloudSolrServer client = createCloudClient("awholynewcollection_" + cnt);
|
||||
try {
|
||||
createCollection(collectionInfos, "awholynewcollection_" + cnt, numShards, replicationFactor, maxShardsPerNode, client);
|
||||
createCollection(collectionInfos, "awholynewcollection_" + cnt, numShards, replicationFactor, maxShardsPerNode, client, null);
|
||||
} finally {
|
||||
client.shutdown();
|
||||
}
|
||||
|
|
@ -922,24 +923,50 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
|
|||
// RESPONSES
|
||||
checkCollectionIsNotCreated(collectionInfos.keySet().iterator().next());
|
||||
|
||||
// Test createNodeSet
|
||||
numLiveNodes = getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size();
|
||||
List<String> createNodeList = new ArrayList<String>();
|
||||
int numOfCreateNodes = numLiveNodes/2;
|
||||
assertFalse("createNodeSet test is pointless with only " + numLiveNodes + " nodes running", numOfCreateNodes == 0);
|
||||
int i = 0;
|
||||
for (String liveNode : getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes()) {
|
||||
if (i < numOfCreateNodes) {
|
||||
createNodeList.add(liveNode);
|
||||
i++;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
maxShardsPerNode = 2;
|
||||
numShards = createNodeList.size() * maxShardsPerNode;
|
||||
replicationFactor = 1;
|
||||
collectionInfos = new HashMap<String,List<Integer>>();
|
||||
client = createCloudClient("awholynewcollection_" + (cnt+1));
|
||||
try {
|
||||
createCollection(collectionInfos, "awholynewcollection_" + (cnt+1), numShards, replicationFactor, maxShardsPerNode, client, StrUtils.join(createNodeList, ','));
|
||||
} finally {
|
||||
client.shutdown();
|
||||
}
|
||||
checkForCollection(collectionInfos.keySet().iterator().next(), collectionInfos.entrySet().iterator().next().getValue(), createNodeList);
|
||||
|
||||
checkNoTwoShardsUseTheSameIndexDir();
|
||||
}
|
||||
|
||||
|
||||
protected void createCollection(Map<String,List<Integer>> collectionInfos,
|
||||
String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client) throws SolrServerException, IOException {
|
||||
String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.set("action", CollectionAction.CREATE.toString());
|
||||
|
||||
params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
|
||||
params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
|
||||
params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
|
||||
if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
|
||||
|
||||
int clientIndex = random().nextInt(2);
|
||||
List<Integer> list = new ArrayList<Integer>();
|
||||
list.add(numShards);
|
||||
list.add(numReplicas);
|
||||
list.add(maxShardsPerNode);
|
||||
collectionInfos.put(collectionName, list);
|
||||
params.set("name", collectionName);
|
||||
SolrRequest request = new QueryRequest(params);
|
||||
|
|
@ -1074,7 +1101,7 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
|
|||
fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
|
||||
}
|
||||
|
||||
private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList) {
|
||||
private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
|
||||
ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
|
||||
|
||||
int expectedSlices = numShardsNumReplicaList.get(0);
|
||||
|
|
@ -1092,6 +1119,11 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
|
|||
}
|
||||
int totalShards = 0;
|
||||
for (String sliceName : slices.keySet()) {
|
||||
for (Replica replica : slices.get(sliceName).getReplicas()) {
|
||||
if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
|
||||
return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
|
||||
}
|
||||
}
|
||||
totalShards += slices.get(sliceName).getReplicas().size();
|
||||
}
|
||||
if (totalShards != expectedTotalShards) {
|
||||
|
|
@ -1103,14 +1135,14 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
|
|||
}
|
||||
}
|
||||
|
||||
private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList)
|
||||
private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
|
||||
throws Exception {
|
||||
// check for an expectedSlices new collection - we poll the state
|
||||
long timeoutAt = System.currentTimeMillis() + 120000;
|
||||
boolean success = false;
|
||||
String checkResult = "Didnt get to perform a single check";
|
||||
while (System.currentTimeMillis() < timeoutAt) {
|
||||
checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList);
|
||||
checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
|
||||
if (checkResult == null) {
|
||||
success = true;
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -19,12 +19,14 @@ package org.apache.solr.cloud;
|
|||
|
||||
import static org.easymock.EasyMock.capture;
|
||||
import static org.easymock.EasyMock.createMock;
|
||||
import static org.easymock.EasyMock.reset;
|
||||
import static org.easymock.EasyMock.expect;
|
||||
import static org.easymock.EasyMock.expectLastCall;
|
||||
import static org.easymock.EasyMock.replay;
|
||||
import static org.easymock.EasyMock.verify;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
|
@ -41,6 +43,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
|
|||
import org.apache.solr.common.params.CoreAdminParams;
|
||||
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.handler.component.ShardHandler;
|
||||
import org.apache.solr.handler.component.ShardRequest;
|
||||
import org.apache.solr.handler.component.ShardResponse;
|
||||
|
|
@ -49,6 +52,7 @@ import org.easymock.IAnswer;
|
|||
import org.eclipse.jetty.util.BlockingArrayQueue;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
||||
|
|
@ -57,12 +61,13 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
private static final String COLLECTION_NAME = "mycollection";
|
||||
private static final String CONFIG_NAME = "myconfig";
|
||||
|
||||
private OverseerCollectionProcessor underTest;
|
||||
private DistributedQueue workQueueMock;
|
||||
private ShardHandler shardHandlerMock;
|
||||
private ZkStateReader zkStateReaderMock;
|
||||
private ClusterState clusterStateMock;
|
||||
private SolrZkClient solrZkClientMock;
|
||||
private static DistributedQueue workQueueMock;
|
||||
private static ShardHandler shardHandlerMock;
|
||||
private static ZkStateReader zkStateReaderMock;
|
||||
private static ClusterState clusterStateMock;
|
||||
private static SolrZkClient solrZkClientMock;
|
||||
|
||||
private OverseerCollectionProcessorToBeTested underTest;
|
||||
|
||||
private Thread thread;
|
||||
private Queue<byte[]> queue = new BlockingArrayQueue<byte[]>();
|
||||
|
|
@ -70,12 +75,20 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
private class OverseerCollectionProcessorToBeTested extends
|
||||
OverseerCollectionProcessor {
|
||||
|
||||
private boolean lastProcessMessageResult = true;
|
||||
|
||||
public OverseerCollectionProcessorToBeTested(ZkStateReader zkStateReader,
|
||||
String myId, ShardHandler shardHandler, String adminPath,
|
||||
DistributedQueue workQueue) {
|
||||
super(zkStateReader, myId, shardHandler, adminPath, workQueue);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean processMessage(ZkNodeProps message, String operation) {
|
||||
lastProcessMessageResult = super.processMessage(message, operation);
|
||||
return lastProcessMessageResult;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean amILeader() {
|
||||
return true;
|
||||
|
|
@ -83,22 +96,30 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
@BeforeClass
|
||||
public static void setUpOnce() throws Exception {
|
||||
workQueueMock = createMock(DistributedQueue.class);
|
||||
shardHandlerMock = createMock(ShardHandler.class);
|
||||
zkStateReaderMock = createMock(ZkStateReader.class);
|
||||
clusterStateMock = createMock(ClusterState.class);
|
||||
solrZkClientMock = createMock(SolrZkClient.class);
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
reset(workQueueMock);
|
||||
reset(workQueueMock);
|
||||
reset(shardHandlerMock);
|
||||
reset(zkStateReaderMock);
|
||||
reset(clusterStateMock);
|
||||
reset(solrZkClientMock);
|
||||
underTest = new OverseerCollectionProcessorToBeTested(zkStateReaderMock,
|
||||
"1234", shardHandlerMock, ADMIN_PATH, workQueueMock);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
underTest.close();
|
||||
thread.interrupt();
|
||||
stopComponentUnderTest();
|
||||
super.tearDown();
|
||||
}
|
||||
|
|
@ -181,6 +202,8 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
protected void stopComponentUnderTest() throws Exception {
|
||||
underTest.close();
|
||||
thread.interrupt();
|
||||
thread.join();
|
||||
}
|
||||
|
||||
|
|
@ -209,25 +232,39 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
protected void issueCreateJob(Integer numberOfSlices,
|
||||
Integer replicationFactor, Integer maxShardsPerNode) {
|
||||
ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION,
|
||||
OverseerCollectionProcessor.CREATECOLLECTION,
|
||||
OverseerCollectionProcessor.REPLICATION_FACTOR,
|
||||
replicationFactor.toString(), "name", COLLECTION_NAME,
|
||||
"collection.configName", CONFIG_NAME,
|
||||
OverseerCollectionProcessor.NUM_SLICES, numberOfSlices.toString(),
|
||||
OverseerCollectionProcessor.MAX_SHARDS_PER_NODE,
|
||||
maxShardsPerNode.toString());
|
||||
Integer replicationFactor, Integer maxShardsPerNode, List<String> createNodeList, boolean sendCreateNodeList) {
|
||||
ZkNodeProps props;
|
||||
if (sendCreateNodeList) {
|
||||
props = new ZkNodeProps(Overseer.QUEUE_OPERATION,
|
||||
OverseerCollectionProcessor.CREATECOLLECTION,
|
||||
OverseerCollectionProcessor.REPLICATION_FACTOR,
|
||||
replicationFactor.toString(), "name", COLLECTION_NAME,
|
||||
"collection.configName", CONFIG_NAME,
|
||||
OverseerCollectionProcessor.NUM_SLICES, numberOfSlices.toString(),
|
||||
OverseerCollectionProcessor.MAX_SHARDS_PER_NODE,
|
||||
maxShardsPerNode.toString(),
|
||||
OverseerCollectionProcessor.CREATE_NODE_SET,
|
||||
(createNodeList != null)?StrUtils.join(createNodeList, ','):null);
|
||||
} else {
|
||||
props = new ZkNodeProps(Overseer.QUEUE_OPERATION,
|
||||
OverseerCollectionProcessor.CREATECOLLECTION,
|
||||
OverseerCollectionProcessor.REPLICATION_FACTOR,
|
||||
replicationFactor.toString(), "name", COLLECTION_NAME,
|
||||
"collection.configName", CONFIG_NAME,
|
||||
OverseerCollectionProcessor.NUM_SLICES, numberOfSlices.toString(),
|
||||
OverseerCollectionProcessor.MAX_SHARDS_PER_NODE,
|
||||
maxShardsPerNode.toString());
|
||||
}
|
||||
queue.add(ZkStateReader.toJSON(props));
|
||||
}
|
||||
|
||||
protected void verifySubmitCaptures(List<SubmitCapture> submitCaptures,
|
||||
Integer numberOfSlices, Integer numberOfReplica, Set<String> liveNodes) {
|
||||
Integer numberOfSlices, Integer numberOfReplica, Collection<String> createNodes) {
|
||||
List<String> coreNames = new ArrayList<String>();
|
||||
Map<String,Map<String,Integer>> sliceToNodeUrlsWithoutProtocolPartToNumberOfShardsRunningMapMap = new HashMap<String,Map<String,Integer>>();
|
||||
List<String> nodeUrlWithoutProtocolPartForLiveNodes = new ArrayList<String>(
|
||||
liveNodes.size());
|
||||
for (String nodeName : liveNodes) {
|
||||
createNodes.size());
|
||||
for (String nodeName : createNodes) {
|
||||
String nodeUrlWithoutProtocolPart = nodeName.replaceAll("_", "/");
|
||||
if (nodeUrlWithoutProtocolPart.startsWith("http://")) nodeUrlWithoutProtocolPart = nodeUrlWithoutProtocolPart
|
||||
.substring(7);
|
||||
|
|
@ -292,13 +329,16 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
sliceToNodeUrlsWithoutProtocolPartToNumberOfShardsRunningMapMap.keySet()
|
||||
.contains("shard" + i);
|
||||
}
|
||||
int minShardsPerSlicePerNode = numberOfReplica / liveNodes.size();
|
||||
int minShardsPerSlicePerNode = numberOfReplica / createNodes.size();
|
||||
int numberOfNodesSupposedToRunMaxShards = numberOfReplica
|
||||
% liveNodes.size();
|
||||
int numberOfNodesSupposedToRunMinShards = liveNodes.size()
|
||||
% createNodes.size();
|
||||
int numberOfNodesSupposedToRunMinShards = createNodes.size()
|
||||
- numberOfNodesSupposedToRunMaxShards;
|
||||
int maxShardsPerSlicePerNode = (numberOfNodesSupposedToRunMaxShards == 0) ? minShardsPerSlicePerNode
|
||||
: (minShardsPerSlicePerNode + 1);
|
||||
int maxShardsPerSlicePerNode = (minShardsPerSlicePerNode + 1);
|
||||
if (numberOfNodesSupposedToRunMaxShards == 0) {
|
||||
numberOfNodesSupposedToRunMaxShards = numberOfNodesSupposedToRunMinShards;
|
||||
maxShardsPerSlicePerNode = minShardsPerSlicePerNode;
|
||||
}
|
||||
boolean diffBetweenMinAndMaxShardsPerSlicePerNode = (maxShardsPerSlicePerNode != minShardsPerSlicePerNode);
|
||||
|
||||
for (Entry<String,Map<String,Integer>> sliceToNodeUrlsWithoutProtocolPartToNumberOfShardsRunningMapMapEntry : sliceToNodeUrlsWithoutProtocolPartToNumberOfShardsRunningMapMap
|
||||
|
|
@ -325,7 +365,7 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
if (numberOfShardsRunningOnThisNode == minShardsPerSlicePerNode) numberOfNodesRunningMinShards++;
|
||||
if (numberOfShardsRunningOnThisNode == maxShardsPerSlicePerNode) numberOfNodesRunningMaxShards++;
|
||||
}
|
||||
if (minShardsPerSlicePerNode == 0) numberOfNodesRunningMinShards = (liveNodes
|
||||
if (minShardsPerSlicePerNode == 0) numberOfNodesRunningMinShards = (createNodes
|
||||
.size() - numberOfNodesRunningAtLeastOneShard);
|
||||
assertEquals(
|
||||
"Too many shards are running under slice "
|
||||
|
|
@ -348,10 +388,25 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
}
|
||||
}
|
||||
|
||||
protected void testTemplate(Integer numberOfNodes, Integer replicationFactor,
|
||||
protected enum CreateNodeListOptions {
|
||||
SEND,
|
||||
DONT_SEND,
|
||||
SEND_NULL
|
||||
}
|
||||
protected void testTemplate(Integer numberOfNodes, Integer numberOfNodesToCreateOn, CreateNodeListOptions createNodeListOption, Integer replicationFactor,
|
||||
Integer numberOfSlices, Integer maxShardsPerNode,
|
||||
boolean collectionExceptedToBeCreated) throws Exception {
|
||||
assertTrue("Wrong usage of testTemplate. numberOfNodesToCreateOn " + numberOfNodesToCreateOn + " is not allowed to be higher than numberOfNodes " + numberOfNodes, numberOfNodes.intValue() >= numberOfNodesToCreateOn.intValue());
|
||||
assertTrue("Wrong usage of testTemplage. createNodeListOption has to be " + CreateNodeListOptions.SEND + " when numberOfNodes and numberOfNodesToCreateOn are unequal", ((createNodeListOption == CreateNodeListOptions.SEND) || (numberOfNodes.intValue() == numberOfNodesToCreateOn.intValue())));
|
||||
|
||||
Set<String> liveNodes = commonMocks(numberOfNodes);
|
||||
List<String> createNodeList = new ArrayList<String>();
|
||||
int i = 0;
|
||||
for (String node : liveNodes) {
|
||||
if (i++ < numberOfNodesToCreateOn) {
|
||||
createNodeList.add(node);
|
||||
}
|
||||
}
|
||||
|
||||
List<SubmitCapture> submitCaptures = null;
|
||||
if (collectionExceptedToBeCreated) {
|
||||
|
|
@ -366,55 +421,112 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
replay(shardHandlerMock);
|
||||
startComponentUnderTest();
|
||||
|
||||
issueCreateJob(numberOfSlices, replicationFactor, maxShardsPerNode);
|
||||
issueCreateJob(numberOfSlices, replicationFactor, maxShardsPerNode, (createNodeListOption != CreateNodeListOptions.SEND_NULL)?createNodeList:null, (createNodeListOption != CreateNodeListOptions.DONT_SEND));
|
||||
|
||||
waitForEmptyQueue(10000);
|
||||
|
||||
assertEquals(collectionExceptedToBeCreated, underTest.lastProcessMessageResult);
|
||||
verify(shardHandlerMock);
|
||||
|
||||
if (collectionExceptedToBeCreated) {
|
||||
verifySubmitCaptures(submitCaptures, numberOfSlices, replicationFactor,
|
||||
liveNodes);
|
||||
createNodeList);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoReplicationEqualNumberOfSlicesPerNode() throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.DONT_SEND;
|
||||
Integer replicationFactor = 1;
|
||||
Integer numberOfSlices = 8;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, replicationFactor, numberOfSlices,
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReplicationEqualNumberOfSlicesPerNode() throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.DONT_SEND;
|
||||
Integer replicationFactor = 2;
|
||||
Integer numberOfSlices = 4;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, replicationFactor, numberOfSlices,
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes() throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.SEND;
|
||||
Integer replicationFactor = 1;
|
||||
Integer numberOfSlices = 8;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes() throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.SEND;
|
||||
Integer replicationFactor = 2;
|
||||
Integer numberOfSlices = 4;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes() throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.SEND_NULL;
|
||||
Integer replicationFactor = 1;
|
||||
Integer numberOfSlices = 8;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes() throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.SEND_NULL;
|
||||
Integer replicationFactor = 2;
|
||||
Integer numberOfSlices = 4;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoReplicationUnequalNumberOfSlicesPerNode() throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.DONT_SEND;
|
||||
Integer replicationFactor = 1;
|
||||
Integer numberOfSlices = 6;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, replicationFactor, numberOfSlices,
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReplicationUnequalNumberOfSlicesPerNode() throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.DONT_SEND;
|
||||
Integer replicationFactor = 2;
|
||||
Integer numberOfSlices = 3;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, replicationFactor, numberOfSlices,
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
|
|
@ -422,10 +534,12 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
public void testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit()
|
||||
throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.DONT_SEND;
|
||||
Integer replicationFactor = 1;
|
||||
Integer numberOfSlices = 6;
|
||||
Integer maxShardsPerNode = 1;
|
||||
testTemplate(numberOfNodes, replicationFactor, numberOfSlices,
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, false);
|
||||
}
|
||||
|
||||
|
|
@ -433,11 +547,65 @@ public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
|
|||
public void testReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit()
|
||||
throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 4;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.DONT_SEND;
|
||||
Integer replicationFactor = 2;
|
||||
Integer numberOfSlices = 3;
|
||||
Integer maxShardsPerNode = 1;
|
||||
testTemplate(numberOfNodes, replicationFactor, numberOfSlices,
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoReplicationLimitedNodesToCreateOn()
|
||||
throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 2;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.SEND;
|
||||
Integer replicationFactor = 1;
|
||||
Integer numberOfSlices = 6;
|
||||
Integer maxShardsPerNode = 3;
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReplicationLimitedNodesToCreateOn()
|
||||
throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 2;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.SEND;
|
||||
Integer replicationFactor = 2;
|
||||
Integer numberOfSlices = 3;
|
||||
Integer maxShardsPerNode = 3;
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits()
|
||||
throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 3;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.SEND;
|
||||
Integer replicationFactor = 1;
|
||||
Integer numberOfSlices = 8;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits()
|
||||
throws Exception {
|
||||
Integer numberOfNodes = 4;
|
||||
Integer numberOfNodesToCreateOn = 3;
|
||||
CreateNodeListOptions createNodeListOptions = CreateNodeListOptions.SEND;
|
||||
Integer replicationFactor = 2;
|
||||
Integer numberOfSlices = 4;
|
||||
Integer maxShardsPerNode = 2;
|
||||
testTemplate(numberOfNodes, numberOfNodesToCreateOn, createNodeListOptions, replicationFactor, numberOfSlices,
|
||||
maxShardsPerNode, false);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue