Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-9132: Migrate some more tests

This commit is contained in:
Alan Woodward 2016-09-27 09:12:45 +01:00
parent 01ecc06915
commit a9dab0f25a
12 changed files with 386 additions and 652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

243
solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
View File

@ -16,127 +16,71 @@
*/
package org.apache.solr.cloud;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrClient;
import java.lang.invoke.MethodHandles;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateAlias;
import org.apache.solr.client.solrj.request.CollectionAdminRequest.DeleteAlias;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.Test;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.List;
/**
* Test sync phase that occurs when Leader goes down and a new Leader is
* elected.
*/
@Slow
public class AliasIntegrationTest extends AbstractFullDistribZkTestBase {
public class AliasIntegrationTest extends SolrCloudTestCase {
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public AliasIntegrationTest() {
super();
sliceCount = 1;
fixShardCount(random().nextBoolean() ? 3 : 4);
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
}
@Test
public void test() throws Exception {
handle.clear();
handle.put("timestamp", SKIPVAL);
CollectionAdminRequest.createCollection("collection1", "conf", 2, 1).process(cluster.getSolrClient());
CollectionAdminRequest.createCollection("collection2", "conf", 1, 1).process(cluster.getSolrClient());
waitForState("Expected collection1 to be created with 2 shards and 1 replica", "collection1", clusterShape(2, 1));
waitForState("Expected collection2 to be created with 1 shard and 1 replica", "collection2", clusterShape(1, 1));
waitForThingsToLevelOut(30);
new UpdateRequest()
.add("id", "6", "a_t", "humpty dumpy sat on a wall")
.add("id", "7", "a_t", "humpty dumpy3 sat on a walls")
.add("id", "8", "a_t", "humpty dumpy2 sat on a walled")
.commit(cluster.getSolrClient(), "collection1");
logger.info("### STARTING ACTUAL TEST");
new UpdateRequest()
.add("id", "9", "a_t", "humpty dumpy sat on a wall")
.add("id", "10", "a_t", "humpty dumpy3 sat on a walls")
.commit(cluster.getSolrClient(), "collection2");
del("*:*");
createCollection("collection2", 2, 1, 10);
List<Integer> numShardsNumReplicaList = new ArrayList<>(2);
numShardsNumReplicaList.add(2);
numShardsNumReplicaList.add(1);
checkForCollection("collection2", numShardsNumReplicaList, null);
waitForRecoveriesToFinish("collection2", true);
cloudClient.setDefaultCollection("collection1");
SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1,
"humpty dumpy sat on a wall");
SolrInputDocument doc2 = getDoc(id, 7, i1, -600, tlong, 600, t1,
"humpty dumpy3 sat on a walls");
SolrInputDocument doc3 = getDoc(id, 8, i1, -600, tlong, 600, t1,
"humpty dumpy2 sat on a walled");
cloudClient.add(doc1);
cloudClient.add(doc2);
cloudClient.add(doc3);
cloudClient.commit();
SolrInputDocument doc6 = getDoc(id, 9, i1, -600, tlong, 600, t1,
"humpty dumpy sat on a wall");
SolrInputDocument doc7 = getDoc(id, 10, i1, -600, tlong, 600, t1,
"humpty dumpy3 sat on a walls");
cloudClient.setDefaultCollection("collection2");
cloudClient.add(doc6);
cloudClient.add(doc7);
cloudClient.commit();
// create alias
createAlias("testalias", "collection1");
CollectionAdminRequest.createAlias("testalias", "collection1").process(cluster.getSolrClient());
// search for alias
SolrQuery query = new SolrQuery("*:*");
query.set("collection", "testalias");
QueryResponse res = cloudClient.query(query);
QueryResponse res = cluster.getSolrClient().query("testalias", new SolrQuery("*:*"));
assertEquals(3, res.getResults().getNumFound());
// search for alias with random non cloud client
query = new SolrQuery("*:*");
query.set("collection", "testalias");
JettySolrRunner jetty = jettys.get(random().nextInt(jettys.size()));
int port = jetty.getLocalPort();
try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) {
res = client.query(query);
JettySolrRunner jetty = cluster.getRandomJetty(random());
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
res = client.query(new SolrQuery("*:*"));
assertEquals(3, res.getResults().getNumFound());
}
// now without collections param
query = new SolrQuery("*:*");
jetty = jettys.get(random().nextInt(jettys.size()));
port = jetty.getLocalPort();
try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) {
res = client.query(query);
assertEquals(3, res.getResults().getNumFound());
}
// create alias, collection2 first because it's not on every node
createAlias("testalias", "collection2,collection1");
CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient());
// search with new cloud client
try (CloudSolrClient cloudSolrClient = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean())) {
try (CloudSolrClient cloudSolrClient = getCloudSolrClient(cluster.getZkServer().getZkAddress(), random().nextBoolean())) {
cloudSolrClient.setParallelUpdates(random().nextBoolean());
query = new SolrQuery("*:*");
SolrQuery query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = cloudSolrClient.query(query);
assertEquals(5, res.getResults().getNumFound());
@ -149,141 +93,76 @@ public class AliasIntegrationTest extends AbstractFullDistribZkTestBase {
}
// search for alias with random non cloud client
query = new SolrQuery("*:*");
query.set("collection", "testalias");
jetty = jettys.get(random().nextInt(jettys.size()));
port = jetty.getLocalPort();
try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) {
jetty = cluster.getRandomJetty(random());
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
SolrQuery query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = client.query(query);
assertEquals(5, res.getResults().getNumFound());
}
// now without collections param
query = new SolrQuery("*:*");
jetty = jettys.get(random().nextInt(jettys.size()));
port = jetty.getLocalPort();
try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) {
// now without collections param
query = new SolrQuery("*:*");
res = client.query(query);
assertEquals(5, res.getResults().getNumFound());
}
// update alias
createAlias("testalias", "collection2");
//checkForAlias("testalias", "collection2");
CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient());
// search for alias
query = new SolrQuery("*:*");
SolrQuery query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = cloudClient.query(query);
res = cluster.getSolrClient().query(query);
assertEquals(2, res.getResults().getNumFound());
// set alias to two collections
createAlias("testalias", "collection1,collection2");
//checkForAlias("testalias", "collection1,collection2");
CollectionAdminRequest.createAlias("testalias", "collection1,collection2").process(cluster.getSolrClient());
query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = cloudClient.query(query);
res = cluster.getSolrClient().query(query);
assertEquals(5, res.getResults().getNumFound());
// try a std client
// search 1 and 2, but have no collections param
query = new SolrQuery("*:*");
try (HttpSolrClient client = getHttpSolrClient(getBaseUrl((HttpSolrClient) clients.get(0)) + "/testalias")) {
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
res = client.query(query);
assertEquals(5, res.getResults().getNumFound());
}
createAlias("testalias", "collection2");
CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient());
// a second alias
createAlias("testalias2", "collection2");
CollectionAdminRequest.createAlias("testalias2", "collection2").process(cluster.getSolrClient());
try (HttpSolrClient client = getHttpSolrClient(getBaseUrl((HttpSolrClient) clients.get(0)) + "/testalias")) {
SolrInputDocument doc8 = getDoc(id, 11, i1, -600, tlong, 600, t1,
"humpty dumpy4 sat on a walls");
client.add(doc8);
client.commit();
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) {
new UpdateRequest()
.add("id", "11", "a_t", "humpty dumpy4 sat on a walls")
.commit(cluster.getSolrClient(), "testalias");
res = client.query(query);
assertEquals(3, res.getResults().getNumFound());
}
createAlias("testalias", "collection2,collection1");
CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient());
query = new SolrQuery("*:*");
query.set("collection", "testalias");
res = cloudClient.query(query);
res = cluster.getSolrClient().query(query);
assertEquals(6, res.getResults().getNumFound());
deleteAlias("testalias");
deleteAlias("testalias2");
CollectionAdminRequest.deleteAlias("testalias").process(cluster.getSolrClient());
CollectionAdminRequest.deleteAlias("testalias2").process(cluster.getSolrClient());
boolean sawException = false;
try {
res = cloudClient.query(query);
} catch (SolrException e) {
sawException = true;
}
assertTrue(sawException);
SolrException e = expectThrows(SolrException.class, () -> {
SolrQuery q = new SolrQuery("*:*");
q.set("collection", "testalias");
cluster.getSolrClient().query(q);
});
assertTrue("Unexpected exception message: " + e.getMessage(), e.getMessage().contains("Collection not found: testalias"));
logger.info("### FINISHED ACTUAL TEST");
}
private void createAlias(String alias, String collections)
throws SolrServerException, IOException {
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
if (random().nextBoolean()) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("collections", collections);
params.set("name", alias);
params.set("action", CollectionAction.CREATEALIAS.toString());
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
client.request(request);
} else {
CreateAlias request = new CreateAlias();
request.setAliasName(alias);
request.setAliasedCollections(collections);
request.process(client);
}
}
}
private void deleteAlias(String alias) throws SolrServerException,
IOException {
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
if (random().nextBoolean()) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("name", alias);
params.set("action", CollectionAction.DELETEALIAS.toString());
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
client.request(request);
} else {
DeleteAlias request = new DeleteAlias();
request.setAliasName(alias);
request.process(client);
}
}
}
protected void indexDoc(List<CloudJettyRunner> skipServers, Object... fields) throws IOException,
SolrServerException {
SolrInputDocument doc = new SolrInputDocument();
addFields(doc, fields);
addFields(doc, "rnd_b", true);
controlClient.add(doc);
UpdateRequest ureq = new UpdateRequest();
ureq.add(doc);
ModifiableSolrParams params = new ModifiableSolrParams();
for (CloudJettyRunner skip : skipServers) {
params.add("test.distrib.skip.servers", skip.url + "/");
}
ureq.setParams(params);
ureq.process(cloudClient);
}
}

44
solr/core/src/test/org/apache/solr/cloud/AsyncCallRequestStatusResponseTest.java
View File

@ -16,29 +16,45 @@
*/
package org.apache.solr.cloud;
import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.common.util.NamedList;
import org.junit.BeforeClass;
import org.junit.Test;
public class AsyncCallRequestStatusResponseTest extends AbstractFullDistribZkTestBase {
public class AsyncCallRequestStatusResponseTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
}
@ShardsFixed(num = 2)
@Test
public void testAsyncCallStatusResponse() throws Exception {
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
create.setCollectionName("asynccall")
.setNumShards(2)
.setAsyncId("1000")
.setConfigName("conf1")
.process(cloudClient);
waitForCollection(cloudClient.getZkStateReader(), "asynccall", 2);
final RequestStatusState state = getRequestStateAfterCompletion("1000", 30, cloudClient);
assertSame(RequestStatusState.COMPLETED, state);
CollectionAdminRequest.RequestStatus requestStatus = new CollectionAdminRequest.RequestStatus();
requestStatus.setRequestId("1000");
CollectionAdminResponse rsp = requestStatus.process(cloudClient);
String asyncId =
CollectionAdminRequest.createCollection("asynccall", "conf", 2, 1).processAsync(cluster.getSolrClient());
waitForState("Expected collection 'asynccall' to have 2 shards and 1 replica", "asynccall", clusterShape(2, 1));
int tries = 0;
while (true) {
final RequestStatusState state
= CollectionAdminRequest.requestStatus(asyncId).process(cluster.getSolrClient()).getRequestStatus();
if (state == RequestStatusState.COMPLETED)
break;
if (tries++ > 10)
fail("Expected to see RequestStatusState.COMPLETED but was " + state.toString());
TimeUnit.SECONDS.sleep(1);
}
CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus(asyncId);
CollectionAdminResponse rsp = requestStatus.process(cluster.getSolrClient());
NamedList<?> r = rsp.getResponse();
// Check that there's more response than the hardcoded status and states
assertEquals("Assertion Failure" + r.toString(), 5, r.size());

121
solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java
View File

@ -1,121 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.junit.Test;
import java.io.IOException;
public class AsyncMigrateRouteKeyTest extends MigrateRouteKeyTest {
public AsyncMigrateRouteKeyTest() {
schemaString = "schema15.xml"; // we need a string id
}
private static final int MAX_WAIT_SECONDS = 2 * 60;
@Test
public void test() throws Exception {
waitForThingsToLevelOut(15);
multipleShardMigrateTest();
printLayout();
}
protected void checkAsyncRequestForCompletion(String asyncId) throws SolrServerException, IOException {
ModifiableSolrParams params;
String message;
params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.REQUESTSTATUS.toString());
params.set(OverseerCollectionMessageHandler.REQUESTID, asyncId);
// This task takes long enough to run. Also check for the current state of the task to be running.
message = sendStatusRequestWithRetry(params, 5);
assertEquals("found [" + asyncId + "] in running tasks", message);
// Now wait until the task actually completes successfully/fails.
message = sendStatusRequestWithRetry(params, MAX_WAIT_SECONDS);
assertEquals("Task " + asyncId + " not found in completed tasks.",
"found [" + asyncId + "] in completed tasks", message);
}
@Override
protected void invokeMigrateApi(String sourceCollection, String splitKey, String targetCollection) throws SolrServerException, IOException {
ModifiableSolrParams params = new ModifiableSolrParams();
String asyncId = "20140128";
params.set(CollectionParams.ACTION, CollectionParams.CollectionAction.MIGRATE.toString());
params.set("collection", sourceCollection);
params.set("target.collection", targetCollection);
params.set("split.key", splitKey);
params.set("forward.timeout", 45);
params.set(CommonAdminParams.ASYNC, asyncId);
invoke(params);
checkAsyncRequestForCompletion(asyncId);
}
/**
* Helper method to send a status request with specific retry limit and return
* the message/null from the success response.
*/
private String sendStatusRequestWithRetry(ModifiableSolrParams params, int maxCounter)
throws SolrServerException, IOException {
NamedList status = null;
RequestStatusState state = null;
String message = null;
NamedList r;
while (maxCounter-- > 0) {
r = sendRequest(params);
status = (NamedList) r.get("status");
state = RequestStatusState.fromKey((String) status.get("state"));
message = (String) status.get("msg");
if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED) {
return (String) status.get("msg");
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
// Return last state?
return message;
}
protected NamedList sendRequest(ModifiableSolrParams params) throws SolrServerException, IOException {
final SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
String baseUrl = ((HttpSolrClient) shardToJetty.get(SHARD1).get(0).client.solrClient).getBaseURL();
baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length());
try (HttpSolrClient baseServer = getHttpSolrClient(baseUrl)) {
baseServer.setConnectionTimeout(15000);
return baseServer.request(request);
}
}
}

81
solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java
View File

@ -19,13 +19,11 @@ package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.RetryUtil;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -33,68 +31,53 @@ import org.slf4j.LoggerFactory;
/**
* Verifies cluster state remains consistent after collection reload.
*/
@Slow
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class CollectionReloadTest extends AbstractFullDistribZkTestBase {
public class CollectionReloadTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public CollectionReloadTest() {
super();
sliceCount = 1;
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(1)
.addConfig("conf", configset("cloud-minimal"))
.configure();
}
@Test
public void testReloadedLeaderStateAfterZkSessionLoss() throws Exception {
waitForThingsToLevelOut(30000);
log.info("testReloadedLeaderStateAfterZkSessionLoss initialized OK ... running test logic");
String testCollectionName = "c8n_1x1";
String shardId = "shard1";
createCollectionRetry(testCollectionName, 1, 1, 1);
cloudClient.setDefaultCollection(testCollectionName);
final String testCollectionName = "c8n_1x1";
CollectionAdminRequest.createCollection(testCollectionName, "conf", 1, 1)
.process(cluster.getSolrClient());
Replica leader = getShardLeader(testCollectionName, shardId, 30 /* timeout secs */);
Replica leader
= cluster.getSolrClient().getZkStateReader().getLeaderRetry(testCollectionName, "shard1", DEFAULT_TIMEOUT);
// reload collection and wait to see the core report it has been reloaded
boolean wasReloaded = reloadCollection(leader, testCollectionName);
assertTrue("Collection '"+testCollectionName+"' failed to reload within a reasonable amount of time!",
wasReloaded);
long coreStartTime = getCoreStatus(leader).getCoreStartTime().getTime();
CollectionAdminRequest.reloadCollection(testCollectionName).process(cluster.getSolrClient());
// cause session loss
chaosMonkey.expireSession(getJettyOnPort(getReplicaPort(leader)));
RetryUtil.retryUntil("Timed out waiting for core to reload", 30, 1000, TimeUnit.MILLISECONDS, () -> {
long restartTime = 0;
try {
restartTime = getCoreStatus(leader).getCoreStartTime().getTime();
} catch (Exception e) {
log.warn("Exception getting core start time: {}", e.getMessage());
return false;
}
return restartTime > coreStartTime;
});
// TODO: have to wait a while for the node to get marked down after ZK session loss
// but tests shouldn't be so timing dependent!
Thread.sleep(15000);
final int initialStateVersion = getCollectionState(testCollectionName).getZNodeVersion();
// wait up to 15 seconds to see the replica in the active state
String replicaState = null;
int timeoutSecs = 15;
long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutSecs, TimeUnit.SECONDS);
while (System.nanoTime() < timeout) {
// state of leader should be active after session loss recovery - see SOLR-7338
cloudClient.getZkStateReader().forceUpdateCollection(testCollectionName);
ClusterState cs = cloudClient.getZkStateReader().getClusterState();
Slice slice = cs.getSlice(testCollectionName, shardId);
replicaState = slice.getReplica(leader.getName()).getStr(ZkStateReader.STATE_PROP);
if ("active".equals(replicaState))
break;
cluster.expireZkSession(cluster.getReplicaJetty(leader));
Thread.sleep(1000);
}
assertEquals("Leader state should be active after recovering from ZK session loss, but after " +
timeoutSecs + " seconds, it is " + replicaState, "active", replicaState);
// try to clean up
try {
new CollectionAdminRequest.Delete()
.setCollectionName(testCollectionName).process(cloudClient);
} catch (Exception e) {
// don't fail the test
log.warn("Could not delete collection {} after test completed", testCollectionName);
}
waitForState("Timed out waiting for core to re-register as ACTIVE after session expiry", testCollectionName, (n, c) -> {
log.info("Collection state: {}", c.toString());
Replica expiredReplica = c.getReplica(leader.getName());
return expiredReplica.getState() == Replica.State.ACTIVE && c.getZNodeVersion() > initialStateVersion;
});
log.info("testReloadedLeaderStateAfterZkSessionLoss succeeded ... shutting down now!");
}

73
solr/core/src/test/org/apache/solr/cloud/CollectionStateFormat2Test.java
View File

@ -16,74 +16,55 @@
*/
package org.apache.solr.cloud;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.zookeeper.data.Stat;
import org.junit.BeforeClass;
import org.junit.Test;
public class CollectionStateFormat2Test extends AbstractFullDistribZkTestBase {
public class CollectionStateFormat2Test extends SolrCloudTestCase {
protected String getSolrXml() {
return "solr.xml";
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(4)
.addConfig("conf", configset("cloud-minimal"))
.configure();
}
@Test
@ShardsFixed(num = 4)
public void test() throws Exception {
try (CloudSolrClient client = createCloudClient(null)) {
testZkNodeLocation(client);
testConfNameAndCollectionNameSame(client);
}
}
public void testConfNameAndCollectionNameSame() throws Exception {
@Override
protected String getStateFormat() {
return "2";
}
private void testConfNameAndCollectionNameSame(CloudSolrClient client) throws Exception{
// .system collection precreates the configset
createCollection(".system", client, 2, 1);
waitForRecoveriesToFinish(".system", false);
CollectionAdminRequest.createCollection(".system", 2, 1)
.process(cluster.getSolrClient());
}
private void testZkNodeLocation(CloudSolrClient client) throws Exception{
@Test
public void testZkNodeLocation() throws Exception {
String collectionName = "myExternColl";
CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2)
.process(cluster.getSolrClient());
createCollection(collectionName, client, 2, 2);
waitForState("Collection not created", collectionName, (n, c) -> DocCollection.isFullyActive(n, c, 2, 2));
assertTrue("State Format 2 collection path does not exist",
zkClient().exists(ZkStateReader.getCollectionPath(collectionName), true));
waitForRecoveriesToFinish(collectionName, false);
assertTrue("does not exist collection state externally",
cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.getCollectionPath(collectionName), true));
Stat stat = new Stat();
byte[] data = cloudClient.getZkStateReader().getZkClient().getData(ZkStateReader.getCollectionPath(collectionName), null, stat, true);
DocCollection c = ZkStateReader.getCollectionLive(cloudClient.getZkStateReader(), collectionName);
ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
assertEquals("The zkversion of the nodes must be same zkver:" + stat.getVersion() , stat.getVersion(),clusterState.getCollection(collectionName).getZNodeVersion() );
assertTrue("DocCllection#getStateFormat() must be > 1", cloudClient.getZkStateReader().getClusterState().getCollection(collectionName).getStateFormat() > 1);
zkClient().getData(ZkStateReader.getCollectionPath(collectionName), null, stat, true);
DocCollection c = getCollectionState(collectionName);
assertEquals("DocCollection version should equal the znode version", stat.getVersion(), c.getZNodeVersion() );
assertTrue("DocCollection#getStateFormat() must be > 1", c.getStateFormat() > 1);
// remove collection
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.DELETE.toString());
params.set("name", collectionName);
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
if (client == null) {
client = createCloudClient(null);
}
CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
waitForState("Collection not deleted", collectionName, (n, coll) -> coll == null);
client.request(request);
assertCollectionNotExists(collectionName, 45);
assertFalse("collection state should not exist externally", cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.getCollectionPath(collectionName), true));
assertFalse("collection state should not exist externally",
zkClient().exists(ZkStateReader.getCollectionPath(collectionName), true));
}
}

156
solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java
View File

@ -16,65 +16,53 @@
*/
package org.apache.solr.cloud;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.RoutingRule;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public class MigrateRouteKeyTest extends SolrCloudTestCase {
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
public class MigrateRouteKeyTest extends BasicDistributedZkTest {
public MigrateRouteKeyTest() {
schemaString = "schema15.xml"; // we need a string id
if (usually()) {
CollectionAdminRequest.setClusterProperty("legacyCloud", "false").process(cluster.getSolrClient());
}
}
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Test
public void test() throws Exception {
waitForThingsToLevelOut(15);
if (usually()) {
log.info("Using legacyCloud=false for cluster");
CollectionsAPIDistributedZkTest.setClusterProp(cloudClient, "legacyCloud", "false");
}
multipleShardMigrateTest();
printLayout();
}
private boolean waitForRuleToExpire(String splitKey, long finishTime) throws KeeperException, InterruptedException, SolrServerException, IOException {
ClusterState state;Slice slice;
private boolean waitForRuleToExpire(String collection, String shard, String splitKey, long finishTime) throws KeeperException, InterruptedException, SolrServerException, IOException {
DocCollection state;
Slice slice;
boolean ruleRemoved = false;
long expiryTime = finishTime + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS);
while (System.nanoTime() < expiryTime) {
getCommonCloudSolrClient().getZkStateReader().forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
state = getCommonCloudSolrClient().getZkStateReader().getClusterState();
slice = state.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD2);
cluster.getSolrClient().getZkStateReader().forceUpdateCollection(collection);
state = getCollectionState(collection);
slice = state.getSlice(shard);
Map<String,RoutingRule> routingRules = slice.getRoutingRules();
if (routingRules == null || routingRules.isEmpty() || !routingRules.containsKey(splitKey)) {
ruleRemoved = true;
@ -82,59 +70,28 @@ public class MigrateRouteKeyTest extends BasicDistributedZkTest {
}
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", splitKey + random().nextInt());
cloudClient.add(doc);
cluster.getSolrClient().add(collection, doc);
Thread.sleep(1000);
}
return ruleRemoved;
}
protected void invokeMigrateApi(String sourceCollection, String splitKey, String targetCollection) throws SolrServerException, IOException {
cloudClient.setDefaultCollection(sourceCollection);
CollectionAdminRequest.Migrate migrateRequest = new CollectionAdminRequest.Migrate();
migrateRequest.setCollectionName(sourceCollection);
migrateRequest.setTargetCollection(targetCollection);
migrateRequest.setSplitKey(splitKey);
migrateRequest.setForwardTimeout(45);
migrateRequest.process(cloudClient);
}
protected void invoke(ModifiableSolrParams params) throws SolrServerException, IOException {
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
String baseUrl = ((HttpSolrClient) shardToJetty.get(SHARD1).get(0).client.solrClient)
.getBaseURL();
baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length());
try (HttpSolrClient baseClient = getHttpSolrClient(baseUrl)) {
baseClient.setConnectionTimeout(15000);
baseClient.setSoTimeout(60000 * 5);
baseClient.request(request);
protected void invokeCollectionMigration(CollectionAdminRequest.AsyncCollectionAdminRequest request) throws IOException, SolrServerException, InterruptedException {
if (random().nextBoolean()) {
cluster.getSolrClient().setSoTimeout(60000); // can take a while
request.process(cluster.getSolrClient());
}
else {
request.processAndWait(cluster.getSolrClient(), 60000);
}
}
private void createCollection(String targetCollection) throws Exception {
HashMap<String, List<Integer>> collectionInfos = new HashMap<>();
@Test
public void multipleShardMigrateTest() throws Exception {
try (CloudSolrClient client = createCloudClient(null)) {
Map<String, Object> props = Utils.makeMap(
REPLICATION_FACTOR, 1,
MAX_SHARDS_PER_NODE, 5,
NUM_SLICES, 1);
CollectionAdminRequest.createCollection("sourceCollection", "conf", 2, 1).process(cluster.getSolrClient());
cluster.getSolrClient().setDefaultCollection("sourceCollection");
createCollection(collectionInfos, targetCollection, props, client);
}
List<Integer> list = collectionInfos.get(targetCollection);
checkForCollection(targetCollection, list, null);
waitForRecoveriesToFinish(targetCollection, false);
}
protected void multipleShardMigrateTest() throws Exception {
del("*:*");
commit();
assertTrue(cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound() == 0);
final String splitKey = "a";
final int BIT_SEP = 1;
final int[] splitKeyCount = new int[1];
@ -147,38 +104,41 @@ public class MigrateRouteKeyTest extends BasicDistributedZkTest {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", key + "!" + id);
doc.addField("n_ti", id);
cloudClient.add(doc);
cluster.getSolrClient().add("sourceCollection", doc);
if (splitKey.equals(shardKey))
splitKeyCount[0]++;
}
assertTrue(splitKeyCount[0] > 0);
String targetCollection = "migrate_multipleshardtest_targetCollection";
createCollection(targetCollection);
CollectionAdminRequest.createCollection(targetCollection, "conf", 1, 1).process(cluster.getSolrClient());
Indexer indexer = new Indexer(cloudClient, splitKey, 1, 30);
Indexer indexer = new Indexer(cluster.getSolrClient(), splitKey, 1, 30);
indexer.start();
String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), targetCollection);
try (HttpSolrClient collectionClient = getHttpSolrClient(url)) {
DocCollection state = getCollectionState(targetCollection);
Replica replica = state.getReplicas().get(0);
try (HttpSolrClient collectionClient = getHttpSolrClient(replica.getCoreUrl())) {
SolrQuery solrQuery = new SolrQuery("*:*");
assertEquals("DocCount on target collection does not match", 0, collectionClient.query(solrQuery).getResults().getNumFound());
invokeMigrateApi(AbstractDistribZkTestBase.DEFAULT_COLLECTION, splitKey + "/" + BIT_SEP + "!", targetCollection);
invokeCollectionMigration(
CollectionAdminRequest.migrateData("sourceCollection", targetCollection, splitKey + "/" + BIT_SEP + "!")
.setForwardTimeout(45));
long finishTime = System.nanoTime();
indexer.join();
splitKeyCount[0] += indexer.getSplitKeyCount();
try {
cloudClient.deleteById("a/" + BIT_SEP + "!104");
cluster.getSolrClient().deleteById("a/" + BIT_SEP + "!104");
splitKeyCount[0]--;
} catch (Exception e) {
log.warn("Error deleting document a/" + BIT_SEP + "!104", e);
}
cloudClient.commit();
cluster.getSolrClient().commit();
collectionClient.commit();
solrQuery = new SolrQuery("*:*").setRows(1000);
@ -186,14 +146,20 @@ public class MigrateRouteKeyTest extends BasicDistributedZkTest {
log.info("Response from target collection: " + response);
assertEquals("DocCount on target collection does not match", splitKeyCount[0], response.getResults().getNumFound());
getCommonCloudSolrClient().getZkStateReader().forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
ClusterState state = getCommonCloudSolrClient().getZkStateReader().getClusterState();
Slice slice = state.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD2);
assertNotNull("Routing rule map is null", slice.getRoutingRules());
assertFalse("Routing rule map is empty", slice.getRoutingRules().isEmpty());
assertNotNull("No routing rule exists for route key: " + splitKey, slice.getRoutingRules().get(splitKey + "!"));
waitForState("Expected to find routing rule for split key " + splitKey, "sourceCollection", (n, c) -> {
if (c == null)
return false;
Slice shard = c.getSlice("shard2");
if (shard == null)
return false;
if (shard.getRoutingRules() == null || shard.getRoutingRules().isEmpty())
return false;
if (shard.getRoutingRules().get(splitKey + "!") == null)
return false;
return true;
});
boolean ruleRemoved = waitForRuleToExpire(splitKey, finishTime);
boolean ruleRemoved = waitForRuleToExpire("sourceCollection", "shard2", splitKey, finishTime);
assertTrue("Routing rule was not expired", ruleRemoved);
}
}
@ -230,7 +196,7 @@ public class MigrateRouteKeyTest extends BasicDistributedZkTest {
try {
Thread.sleep(50);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
}
}
}

209
solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
View File

@ -17,23 +17,19 @@
package org.apache.solr.cloud.rule;
import java.lang.invoke.MethodHandles;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.slf4j.Logger;
@ -43,34 +39,35 @@ import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
import static org.junit.matchers.JUnitMatchers.containsString;
public class RulesTest extends AbstractFullDistribZkTestBase {
@LuceneTestCase.Slow
public class RulesTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(5)
.addConfig("conf", configset("cloud-minimal"))
.configure();
}
@org.junit.Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
@ShardsFixed(num = 5)
public void doIntegrationTest() throws Exception {
final long minGB = (random().nextBoolean() ? 1 : 0);
assumeTrue("doIntegrationTest needs minGB="+minGB+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB);
assumeTrue("doIntegrationTest needs minGB="+minGB+" usable disk space",
ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB);
String rulesColl = "rulesColl";
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
CollectionAdminResponse rsp;
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create()
.setCollectionName(rulesColl)
.setShards("shard1")
.setRouterName(ImplicitDocRouter.NAME)
.setReplicationFactor(2)
.setRule("cores:<4", "node:*,replica:<2", "freedisk:>"+minGB)
.setSnitch("class:ImplicitSnitch");
rsp = create.process(client);
assertEquals(0, rsp.getStatus());
assertTrue(rsp.isSuccess());
CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
.setRule("cores:<4", "node:*,replica:<2", "freedisk:>"+minGB)
.setSnitch("class:ImplicitSnitch")
.process(cluster.getSolrClient());
}
DocCollection rulesCollection = getCollectionState(rulesColl);
DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl);
List list = (List) rulesCollection.get("rule");
assertEquals(3, list.size());
assertEquals ( "<4", ((Map)list.get(0)).get("cores"));
@ -80,51 +77,25 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
assertEquals(1, list.size());
assertEquals ( "ImplicitSnitch", ((Map)list.get(0)).get("class"));
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
CollectionAdminResponse rsp;
CollectionAdminRequest.CreateShard createShard = new CollectionAdminRequest.CreateShard()
.setCollectionName(rulesColl)
.setShardName("shard2");
rsp = createShard.process(client);
assertEquals(0, rsp.getStatus());
assertTrue(rsp.isSuccess());
CollectionAdminRequest.AddReplica addReplica = new CollectionAdminRequest.AddReplica()
.setCollectionName(rulesColl)
.setShardName("shard2");
rsp = addReplica.process(client);
assertEquals(0, rsp.getStatus());
assertTrue(rsp.isSuccess());
}
CollectionAdminRequest.createShard(rulesColl, "shard2").process(cluster.getSolrClient());
CollectionAdminRequest.addReplicaToShard(rulesColl, "shard2").process(cluster.getSolrClient());
}
@Test
public void testPortRule() throws Exception {
JettySolrRunner jetty = cluster.getRandomJetty(random());
String port = Integer.toString(jetty.getLocalPort());
String rulesColl = "portRuleColl";
String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
String port = "-1";
Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
if (hostAndPortMatcher.find()) {
port = hostAndPortMatcher.group(2);
}
try (SolrClient client = createNewSolrClient("", baseUrl)) {
CollectionAdminResponse rsp;
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
create.setCollectionName(rulesColl);
create.setShards("shard1");
create.setRouterName(ImplicitDocRouter.NAME);
create.setReplicationFactor(2);
create.setRule("port:" + port);
create.setSnitch("class:ImplicitSnitch");
rsp = create.process(client);
assertEquals(0, rsp.getStatus());
assertTrue(rsp.isSuccess());
CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
.setRule("port:" + port)
.setSnitch("class:ImplicitSnitch")
.process(cluster.getSolrClient());
}
DocCollection rulesCollection = getCollectionState(rulesColl);
DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl);
List list = (List) rulesCollection.get("rule");
assertEquals(1, list.size());
assertEquals(port, ((Map) list.get(0)).get("port"));
@ -135,33 +106,21 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
@Test
public void testHostFragmentRule() throws Exception {
String rulesColl = "ipRuleColl";
String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
String ip_1 = "-1";
String ip_2 = "-1";
Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
if (hostAndPortMatcher.find()) {
String[] ipFragments = hostAndPortMatcher.group(1).split("\\.");
ip_1 = ipFragments[ipFragments.length - 1];
ip_2 = ipFragments[ipFragments.length - 2];
}
try (SolrClient client = createNewSolrClient("", baseUrl)) {
CollectionAdminResponse rsp;
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
create.setCollectionName(rulesColl);
create.setShards("shard1");
create.setRouterName(ImplicitDocRouter.NAME);
create.setReplicationFactor(2);
create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1);
create.setSnitch("class:ImplicitSnitch");
rsp = create.process(client);
assertEquals(0, rsp.getStatus());
assertTrue(rsp.isSuccess());
String rulesColl = "hostFragment";
}
JettySolrRunner jetty = cluster.getRandomJetty(random());
String host = jetty.getBaseUrl().getHost();
String[] ipFragments = host.split("\\.");
String ip_1 = ipFragments[ipFragments.length - 1];
String ip_2 = ipFragments[ipFragments.length - 2];
DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl);
CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
.setRule("ip_2:" + ip_2, "ip_1:" + ip_1)
.setSnitch("class:ImplicitSnitch")
.process(cluster.getSolrClient());
DocCollection rulesCollection = getCollectionState(rulesColl);
List<Map> list = (List<Map>) rulesCollection.get("rule");
assertEquals(2, list.size());
assertEquals(ip_2, list.get(0).get("ip_2"));
@ -175,66 +134,54 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
@Test
public void testHostFragmentRuleThrowsExceptionWhenIpDoesNotMatch() throws Exception {
String rulesColl = "ipRuleColl";
String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
String ip_1 = "-1";
String ip_2 = "-1";
Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
if (hostAndPortMatcher.find()) {
String[] ipFragments = hostAndPortMatcher.group(1).split("\\.");
ip_1 = ipFragments[ipFragments.length - 1];
ip_2 = ipFragments[ipFragments.length - 2];
}
try (SolrClient client = createNewSolrClient("", baseUrl)) {
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
create.setCollectionName(rulesColl);
create.setShards("shard1");
create.setRouterName(ImplicitDocRouter.NAME);
create.setReplicationFactor(2);
JettySolrRunner jetty = cluster.getRandomJetty(random());
String host = jetty.getBaseUrl().getHost();
String[] ipFragments = host.split("\\.");
String ip_1 = ipFragments[ipFragments.length - 1];
String ip_2 = ipFragments[ipFragments.length - 2];
create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1 + "9999");
create.setSnitch("class:ImplicitSnitch");
expectedException.expect(HttpSolrClient.RemoteSolrException.class);
expectedException.expectMessage(containsString("ip_1"));
expectedException.expect(HttpSolrClient.RemoteSolrException.class);
expectedException.expectMessage(containsString("ip_1"));
create.process(client);
}
CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
.setRule("ip_2:" + ip_2, "ip_1:" + ip_1 + "9999")
.setSnitch("class:ImplicitSnitch")
.process(cluster.getSolrClient());
}
@Test
public void testModifyColl() throws Exception {
final long minGB1 = (random().nextBoolean() ? 1 : 0);
final long minGB2 = 5;
assumeTrue("testModifyColl needs minGB1="+minGB1+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB1);
assumeTrue("testModifyColl needs minGB2="+minGB2+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB2);
assumeTrue("testModifyColl needs minGB1="+minGB1+" usable disk space",
ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB1);
assumeTrue("testModifyColl needs minGB2="+minGB2+" usable disk space",
ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB2);
String rulesColl = "modifyColl";
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
CollectionAdminResponse rsp;
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create()
.setCollectionName(rulesColl)
.setNumShards(1)
.setReplicationFactor(2)
.setRule("cores:<4", "node:*,replica:1", "freedisk:>"+minGB1)
.setSnitch("class:ImplicitSnitch");
rsp = create.process(client);
assertEquals(0, rsp.getStatus());
assertTrue(rsp.isSuccess());
ModifiableSolrParams p = new ModifiableSolrParams();
p.add("collection", rulesColl);
p.add("action", "MODIFYCOLLECTION");
p.add("rule", "cores:<5");
p.add("rule", "node:*,replica:1");
p.add("rule", "freedisk:>"+minGB2);
p.add("autoAddReplicas", "true");
client.request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
}
CollectionAdminRequest.createCollection(rulesColl, "conf", 1, 2)
.setRule("cores:<4", "node:*,replica:1", "freedisk:>" + minGB1)
.setSnitch("class:ImplicitSnitch")
.process(cluster.getSolrClient());
DocCollection rulesCollection = ZkStateReader.getCollectionLive(cloudClient.getZkStateReader(), rulesColl);
// TODO: Make a MODIFYCOLLECTION SolrJ class
ModifiableSolrParams p = new ModifiableSolrParams();
p.add("collection", rulesColl);
p.add("action", "MODIFYCOLLECTION");
p.add("rule", "cores:<5");
p.add("rule", "node:*,replica:1");
p.add("rule", "freedisk:>"+minGB2);
p.add("autoAddReplicas", "true");
cluster.getSolrClient().request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
DocCollection rulesCollection = getCollectionState(rulesColl);
log.info("version_of_coll {} ", rulesCollection.getZNodeVersion());
List list = (List) rulesCollection.get("rule");
assertEquals(3, list.size());

52
solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
View File

@ -32,6 +32,7 @@ import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.client.solrj.util.SolrIdentifierValidator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
@ -319,6 +320,31 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return new Create(collection, config, numShards, numReplicas);
}
/**
* Returns a SolrRequest for creating a collection using a default configSet
*
* This requires that there is either a single configset configured in the cluster, or
* that there is a configset with the same name as the collection
*
* @param collection the collection name
* @param numShards the number of shards in the collection
* @param numReplicas the replication factor of the collection
*/
public static Create createCollection(String collection, int numShards, int numReplicas) {
return new Create(collection, numShards, numReplicas);
}
/**
* Returns a SolrRequest for creating a collection with the implicit router
* @param collection the collection name
* @param config the collection config
* @param shards a shard definition string
* @param numReplicas the replication factor of the collection
*/
public static Create createCollectionWithImplicitRouter(String collection, String config, String shards, int numReplicas) {
return new Create(collection, config, shards, numReplicas);
}
// CREATE request
public static class Create extends AsyncCollectionSpecificAdminRequest {
@ -351,6 +377,20 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
this.replicationFactor = numReplicas;
}
private Create(String collection, int numShards, int numReplicas) {
super(CollectionAction.CREATE, SolrIdentifierValidator.validateCollectionName(collection));
this.numShards = numShards;
this.replicationFactor = numReplicas;
}
private Create(String collection, String config, String shards, int numReplicas) {
super(CollectionAction.CREATE, SolrIdentifierValidator.validateCollectionName(collection));
this.configName = config;
this.replicationFactor = numReplicas;
this.shards = shards;
this.routerName = ImplicitDocRouter.NAME;
}
@Deprecated
public Create setConfigName(String config) { this.configName = config; return this; }
public Create setCreateNodeSet(String nodeSet) { this.createNodeSet = nodeSet; return this; }
@ -424,16 +464,20 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.set("collection.configName", configName);
params.set("createNodeSet", createNodeSet);
if (configName != null)
params.set("collection.configName", configName);
if (createNodeSet != null)
params.set("createNodeSet", createNodeSet);
if (numShards != null) {
params.set( ZkStateReader.NUM_SHARDS_PROP, numShards);
}
if (maxShardsPerNode != null) {
params.set( "maxShardsPerNode", maxShardsPerNode);
}
params.set( "router.name", routerName);
params.set("shards", shards);
if (routerName != null)
params.set( "router.name", routerName);
if (shards != null)
params.set("shards", shards);
if (routerField != null) {
params.set("router.field", routerField);
}

6
solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreStatus.java
View File

@ -17,6 +17,8 @@
package org.apache.solr.client.solrj.request;
import java.util.Date;
import org.apache.solr.common.util.NamedList;
public class CoreStatus {
@ -39,4 +41,8 @@ public class CoreStatus {
public String toString() {
return response.toString();
}
public Date getCoreStartTime() {
return (Date) response.get("startTime");
}
}

11
solr/solrj/src/java/org/apache/solr/common/util/RetryUtil.java
View File

@ -19,7 +19,6 @@ package org.apache.solr.common.util;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.solr.common.SolrException;
@ -69,6 +68,16 @@ public class RetryUtil {
return false;
}
public static void retryUntil(String errorMessage, int retries, long pauseTime, TimeUnit pauseUnit, BooleanRetryCmd cmd)
throws InterruptedException {
while (retries-- > 0) {
if (cmd.execute())
return;
pauseUnit.sleep(pauseTime);
}
throw new SolrException(ErrorCode.SERVER_ERROR, errorMessage);
}
public static void retryOnBoolean(long timeoutms, long intervalms, BooleanRetryCmd cmd) {
long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutms, TimeUnit.MILLISECONDS);
while (true) {

22
solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.cloud;
import javax.servlet.Filter;
import java.io.File;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
@ -57,14 +58,19 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.KeeperException;
import org.eclipse.jetty.servlet.ServletHolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* "Mini" SolrCloud cluster to be used for testing
*/
public class MiniSolrCloudCluster {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String DEFAULT_CLOUD_SOLR_XML = "<solr>\n" +
"\n" +
" <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
@ -187,6 +193,8 @@ public class MiniSolrCloudCluster {
this.baseDir = Objects.requireNonNull(baseDir);
this.jettyConfig = Objects.requireNonNull(jettyConfig);
log.info("Starting cluster of {} servers in {}", numServers, baseDir);
Files.createDirectories(baseDir);
this.externalZkServer = zkTestServer != null;
@ -513,4 +521,18 @@ public class MiniSolrCloudCluster {
}
throw new IllegalArgumentException("Cannot find Jetty for a replica with core url " + replica.getCoreUrl());
}
/**
* Make the zookeeper session on a particular jetty expire
*/
public void expireZkSession(JettySolrRunner jetty) {
CoreContainer cores = jetty.getCoreContainer();
if (cores != null) {
SolrZkClient zkClient = cores.getZkController().getZkClient();
zkClient.getSolrZooKeeper().closeCnxn();
long sessionId = zkClient.getSolrZooKeeper().getSessionId();
zkServer.expire(sessionId);
log.info("Expired zookeeper session {} from node {}", sessionId, jetty.getBaseUrl());
}
}
}

12
solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
View File

@ -229,7 +229,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
return predicate.matches(n, c);
});
} catch (Exception e) {
fail(message + "\nLast available state: " + state.get());
fail(message + "\n" + e.getMessage() + "\nLast available state: " + state.get());
}
}
@ -239,6 +239,8 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
*/
public static CollectionStatePredicate clusterShape(int expectedShards, int expectedReplicas) {
return (liveNodes, collectionState) -> {
if (collectionState == null)
return false;
if (collectionState.getSlices().size() != expectedShards)
return false;
for (Slice slice : collectionState) {
@ -257,7 +259,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
/**
* Get a (reproducibly) random shard from a {@link DocCollection}
*/
protected Slice getRandomShard(DocCollection collection) {
protected static Slice getRandomShard(DocCollection collection) {
List<Slice> shards = new ArrayList<>(collection.getActiveSlices());
if (shards.size() == 0)
fail("Couldn't get random shard for collection as it has no shards!\n" + collection.toString());
@ -268,7 +270,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
/**
* Get a (reproducibly) random replica from a {@link Slice}
*/
protected Replica getRandomReplica(Slice slice) {
protected static Replica getRandomReplica(Slice slice) {
List<Replica> replicas = new ArrayList<>(slice.getReplicas());
if (replicas.size() == 0)
fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString());
@ -279,7 +281,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
/**
* Get a (reproducibly) random replica from a {@link Slice} matching a predicate
*/
protected Replica getRandomReplica(Slice slice, Predicate<Replica> matchPredicate) {
protected static Replica getRandomReplica(Slice slice, Predicate<Replica> matchPredicate) {
List<Replica> replicas = new ArrayList<>(slice.getReplicas());
if (replicas.size() == 0)
fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString());
@ -297,7 +299,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
*
* This assumes that the replica is hosted on a live node.
*/
protected CoreStatus getCoreStatus(Replica replica) throws IOException, SolrServerException {
protected static CoreStatus getCoreStatus(Replica replica) throws IOException, SolrServerException {
JettySolrRunner jetty = cluster.getReplicaJetty(replica);
try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString(), cluster.getSolrClient().getHttpClient())) {
return CoreAdminRequest.getCoreStatus(replica.getCoreName(), client);