From a9dab0f25aaea903fe846f91ff0968f16b685851 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 27 Sep 2016 09:12:45 +0100 Subject: [PATCH] SOLR-9132: Migrate some more tests --- .../solr/cloud/AliasIntegrationTest.java | 251 +++++------------- .../AsyncCallRequestStatusResponseTest.java | 44 ++- .../solr/cloud/AsyncMigrateRouteKeyTest.java | 121 --------- .../solr/cloud/CollectionReloadTest.java | 81 +++--- .../cloud/CollectionStateFormat2Test.java | 73 ++--- .../solr/cloud/MigrateRouteKeyTest.java | 156 +++++------ .../org/apache/solr/cloud/rule/RulesTest.java | 209 ++++++--------- .../solrj/request/CollectionAdminRequest.java | 52 +++- .../solr/client/solrj/request/CoreStatus.java | 6 + .../apache/solr/common/util/RetryUtil.java | 11 +- .../solr/cloud/MiniSolrCloudCluster.java | 22 ++ .../apache/solr/cloud/SolrCloudTestCase.java | 12 +- 12 files changed, 386 insertions(+), 652 deletions(-) delete mode 100644 solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java diff --git a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java index 18172e9e415..50e84e45d90 100644 --- a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java @@ -16,127 +16,71 @@ */ package org.apache.solr.cloud; -import org.apache.lucene.util.LuceneTestCase.Slow; -import org.apache.solr.client.solrj.SolrClient; +import java.lang.invoke.MethodHandles; + import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateAlias; -import org.apache.solr.client.solrj.request.CollectionAdminRequest.DeleteAlias; -import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.BeforeClass; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.junit.Test; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; - -/** - * Test sync phase that occurs when Leader goes down and a new Leader is - * elected. - */ -@Slow -public class AliasIntegrationTest extends AbstractFullDistribZkTestBase { +public class AliasIntegrationTest extends SolrCloudTestCase { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public AliasIntegrationTest() { - super(); - sliceCount = 1; - fixShardCount(random().nextBoolean() ? 3 : 4); + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(2) + .addConfig("conf", configset("cloud-minimal")) + .configure(); } @Test public void test() throws Exception { - handle.clear(); - handle.put("timestamp", SKIPVAL); + CollectionAdminRequest.createCollection("collection1", "conf", 2, 1).process(cluster.getSolrClient()); + CollectionAdminRequest.createCollection("collection2", "conf", 1, 1).process(cluster.getSolrClient()); + waitForState("Expected collection1 to be created with 2 shards and 1 replica", "collection1", clusterShape(2, 1)); + waitForState("Expected collection2 to be created with 1 shard and 1 replica", "collection2", clusterShape(1, 1)); - waitForThingsToLevelOut(30); + new UpdateRequest() + .add("id", "6", "a_t", "humpty dumpy sat on a wall") + .add("id", "7", "a_t", "humpty dumpy3 sat on a walls") + .add("id", "8", "a_t", "humpty dumpy2 sat on a walled") + .commit(cluster.getSolrClient(), "collection1"); - logger.info("### STARTING ACTUAL TEST"); + new UpdateRequest() + .add("id", "9", "a_t", "humpty dumpy sat on a wall") + .add("id", "10", "a_t", "humpty dumpy3 sat on a walls") + .commit(cluster.getSolrClient(), "collection2"); - del("*:*"); - - createCollection("collection2", 2, 1, 10); - - List numShardsNumReplicaList = new ArrayList<>(2); - numShardsNumReplicaList.add(2); - numShardsNumReplicaList.add(1); - checkForCollection("collection2", numShardsNumReplicaList, null); - waitForRecoveriesToFinish("collection2", true); - - cloudClient.setDefaultCollection("collection1"); - - SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1, - "humpty dumpy sat on a wall"); - SolrInputDocument doc2 = getDoc(id, 7, i1, -600, tlong, 600, t1, - "humpty dumpy3 sat on a walls"); - SolrInputDocument doc3 = getDoc(id, 8, i1, -600, tlong, 600, t1, - "humpty dumpy2 sat on a walled"); + CollectionAdminRequest.createAlias("testalias", "collection1").process(cluster.getSolrClient()); - cloudClient.add(doc1); - cloudClient.add(doc2); - cloudClient.add(doc3); - - cloudClient.commit(); - - SolrInputDocument doc6 = getDoc(id, 9, i1, -600, tlong, 600, t1, - "humpty dumpy sat on a wall"); - SolrInputDocument doc7 = getDoc(id, 10, i1, -600, tlong, 600, t1, - "humpty dumpy3 sat on a walls"); - - cloudClient.setDefaultCollection("collection2"); - - cloudClient.add(doc6); - cloudClient.add(doc7); - - cloudClient.commit(); - - // create alias - createAlias("testalias", "collection1"); - // search for alias - SolrQuery query = new SolrQuery("*:*"); - query.set("collection", "testalias"); - QueryResponse res = cloudClient.query(query); + QueryResponse res = cluster.getSolrClient().query("testalias", new SolrQuery("*:*")); assertEquals(3, res.getResults().getNumFound()); // search for alias with random non cloud client - query = new SolrQuery("*:*"); - query.set("collection", "testalias"); - JettySolrRunner jetty = jettys.get(random().nextInt(jettys.size())); - int port = jetty.getLocalPort(); - try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) { - res = client.query(query); + JettySolrRunner jetty = cluster.getRandomJetty(random()); + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) { + res = client.query(new SolrQuery("*:*")); assertEquals(3, res.getResults().getNumFound()); } - // now without collections param - query = new SolrQuery("*:*"); - jetty = jettys.get(random().nextInt(jettys.size())); - port = jetty.getLocalPort(); - try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) { - res = client.query(query); - assertEquals(3, res.getResults().getNumFound()); - } // create alias, collection2 first because it's not on every node - createAlias("testalias", "collection2,collection1"); + CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient()); // search with new cloud client - try (CloudSolrClient cloudSolrClient = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean())) { + try (CloudSolrClient cloudSolrClient = getCloudSolrClient(cluster.getZkServer().getZkAddress(), random().nextBoolean())) { cloudSolrClient.setParallelUpdates(random().nextBoolean()); - query = new SolrQuery("*:*"); + SolrQuery query = new SolrQuery("*:*"); query.set("collection", "testalias"); res = cloudSolrClient.query(query); assertEquals(5, res.getResults().getNumFound()); @@ -149,141 +93,76 @@ public class AliasIntegrationTest extends AbstractFullDistribZkTestBase { } // search for alias with random non cloud client - query = new SolrQuery("*:*"); - query.set("collection", "testalias"); - jetty = jettys.get(random().nextInt(jettys.size())); - port = jetty.getLocalPort(); - try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) { + jetty = cluster.getRandomJetty(random()); + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) { + SolrQuery query = new SolrQuery("*:*"); + query.set("collection", "testalias"); res = client.query(query); assertEquals(5, res.getResults().getNumFound()); - } - // now without collections param - query = new SolrQuery("*:*"); - jetty = jettys.get(random().nextInt(jettys.size())); - port = jetty.getLocalPort(); - try (HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/testalias")) { + + // now without collections param + query = new SolrQuery("*:*"); res = client.query(query); assertEquals(5, res.getResults().getNumFound()); } // update alias - createAlias("testalias", "collection2"); - //checkForAlias("testalias", "collection2"); - + CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient()); + // search for alias - query = new SolrQuery("*:*"); + SolrQuery query = new SolrQuery("*:*"); query.set("collection", "testalias"); - res = cloudClient.query(query); + res = cluster.getSolrClient().query(query); assertEquals(2, res.getResults().getNumFound()); // set alias to two collections - createAlias("testalias", "collection1,collection2"); - //checkForAlias("testalias", "collection1,collection2"); - + CollectionAdminRequest.createAlias("testalias", "collection1,collection2").process(cluster.getSolrClient()); + query = new SolrQuery("*:*"); query.set("collection", "testalias"); - res = cloudClient.query(query); + res = cluster.getSolrClient().query(query); assertEquals(5, res.getResults().getNumFound()); // try a std client // search 1 and 2, but have no collections param query = new SolrQuery("*:*"); - try (HttpSolrClient client = getHttpSolrClient(getBaseUrl((HttpSolrClient) clients.get(0)) + "/testalias")) { + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) { res = client.query(query); assertEquals(5, res.getResults().getNumFound()); } - createAlias("testalias", "collection2"); + CollectionAdminRequest.createAlias("testalias", "collection2").process(cluster.getSolrClient()); // a second alias - createAlias("testalias2", "collection2"); + CollectionAdminRequest.createAlias("testalias2", "collection2").process(cluster.getSolrClient()); - try (HttpSolrClient client = getHttpSolrClient(getBaseUrl((HttpSolrClient) clients.get(0)) + "/testalias")) { - SolrInputDocument doc8 = getDoc(id, 11, i1, -600, tlong, 600, t1, - "humpty dumpy4 sat on a walls"); - client.add(doc8); - client.commit(); + try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString() + "/testalias")) { + new UpdateRequest() + .add("id", "11", "a_t", "humpty dumpy4 sat on a walls") + .commit(cluster.getSolrClient(), "testalias"); res = client.query(query); assertEquals(3, res.getResults().getNumFound()); } - - createAlias("testalias", "collection2,collection1"); + + CollectionAdminRequest.createAlias("testalias", "collection2,collection1").process(cluster.getSolrClient()); query = new SolrQuery("*:*"); query.set("collection", "testalias"); - res = cloudClient.query(query); + res = cluster.getSolrClient().query(query); assertEquals(6, res.getResults().getNumFound()); - - deleteAlias("testalias"); - deleteAlias("testalias2"); - boolean sawException = false; - try { - res = cloudClient.query(query); - } catch (SolrException e) { - sawException = true; - } - assertTrue(sawException); + CollectionAdminRequest.deleteAlias("testalias").process(cluster.getSolrClient()); + CollectionAdminRequest.deleteAlias("testalias2").process(cluster.getSolrClient()); + + SolrException e = expectThrows(SolrException.class, () -> { + SolrQuery q = new SolrQuery("*:*"); + q.set("collection", "testalias"); + cluster.getSolrClient().query(q); + }); + assertTrue("Unexpected exception message: " + e.getMessage(), e.getMessage().contains("Collection not found: testalias")); logger.info("### FINISHED ACTUAL TEST"); } - private void createAlias(String alias, String collections) - throws SolrServerException, IOException { - - try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) { - if (random().nextBoolean()) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("collections", collections); - params.set("name", alias); - params.set("action", CollectionAction.CREATEALIAS.toString()); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - client.request(request); - } else { - CreateAlias request = new CreateAlias(); - request.setAliasName(alias); - request.setAliasedCollections(collections); - request.process(client); - } - } - } - - private void deleteAlias(String alias) throws SolrServerException, - IOException { - try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) { - if (random().nextBoolean()) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("name", alias); - params.set("action", CollectionAction.DELETEALIAS.toString()); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - client.request(request); - } else { - DeleteAlias request = new DeleteAlias(); - request.setAliasName(alias); - request.process(client); - } - } - } - - protected void indexDoc(List skipServers, Object... fields) throws IOException, - SolrServerException { - SolrInputDocument doc = new SolrInputDocument(); - - addFields(doc, fields); - addFields(doc, "rnd_b", true); - - controlClient.add(doc); - - UpdateRequest ureq = new UpdateRequest(); - ureq.add(doc); - ModifiableSolrParams params = new ModifiableSolrParams(); - for (CloudJettyRunner skip : skipServers) { - params.add("test.distrib.skip.servers", skip.url + "/"); - } - ureq.setParams(params); - ureq.process(cloudClient); - } } diff --git a/solr/core/src/test/org/apache/solr/cloud/AsyncCallRequestStatusResponseTest.java b/solr/core/src/test/org/apache/solr/cloud/AsyncCallRequestStatusResponseTest.java index 71b6b44f051..7464c87f62e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/AsyncCallRequestStatusResponseTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/AsyncCallRequestStatusResponseTest.java @@ -16,29 +16,45 @@ */ package org.apache.solr.cloud; +import java.util.concurrent.TimeUnit; + import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.common.util.NamedList; +import org.junit.BeforeClass; import org.junit.Test; -public class AsyncCallRequestStatusResponseTest extends AbstractFullDistribZkTestBase { +public class AsyncCallRequestStatusResponseTest extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(2) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } - @ShardsFixed(num = 2) @Test public void testAsyncCallStatusResponse() throws Exception { - CollectionAdminRequest.Create create = new CollectionAdminRequest.Create(); - create.setCollectionName("asynccall") - .setNumShards(2) - .setAsyncId("1000") - .setConfigName("conf1") - .process(cloudClient); - waitForCollection(cloudClient.getZkStateReader(), "asynccall", 2); - final RequestStatusState state = getRequestStateAfterCompletion("1000", 30, cloudClient); - assertSame(RequestStatusState.COMPLETED, state); - CollectionAdminRequest.RequestStatus requestStatus = new CollectionAdminRequest.RequestStatus(); - requestStatus.setRequestId("1000"); - CollectionAdminResponse rsp = requestStatus.process(cloudClient); + + String asyncId = + CollectionAdminRequest.createCollection("asynccall", "conf", 2, 1).processAsync(cluster.getSolrClient()); + + waitForState("Expected collection 'asynccall' to have 2 shards and 1 replica", "asynccall", clusterShape(2, 1)); + + int tries = 0; + while (true) { + final RequestStatusState state + = CollectionAdminRequest.requestStatus(asyncId).process(cluster.getSolrClient()).getRequestStatus(); + if (state == RequestStatusState.COMPLETED) + break; + if (tries++ > 10) + fail("Expected to see RequestStatusState.COMPLETED but was " + state.toString()); + TimeUnit.SECONDS.sleep(1); + } + + CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus(asyncId); + CollectionAdminResponse rsp = requestStatus.process(cluster.getSolrClient()); NamedList r = rsp.getResponse(); // Check that there's more response than the hardcoded status and states assertEquals("Assertion Failure" + r.toString(), 5, r.size()); diff --git a/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java deleted file mode 100644 index 7c71a92d1cf..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonAdminParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.NamedList; -import org.junit.Test; - -import java.io.IOException; - -public class AsyncMigrateRouteKeyTest extends MigrateRouteKeyTest { - - public AsyncMigrateRouteKeyTest() { - schemaString = "schema15.xml"; // we need a string id - } - - private static final int MAX_WAIT_SECONDS = 2 * 60; - - @Test - public void test() throws Exception { - waitForThingsToLevelOut(15); - - multipleShardMigrateTest(); - printLayout(); - } - - protected void checkAsyncRequestForCompletion(String asyncId) throws SolrServerException, IOException { - ModifiableSolrParams params; - String message; - params = new ModifiableSolrParams(); - params.set("action", CollectionParams.CollectionAction.REQUESTSTATUS.toString()); - params.set(OverseerCollectionMessageHandler.REQUESTID, asyncId); - // This task takes long enough to run. Also check for the current state of the task to be running. - message = sendStatusRequestWithRetry(params, 5); - assertEquals("found [" + asyncId + "] in running tasks", message); - // Now wait until the task actually completes successfully/fails. - message = sendStatusRequestWithRetry(params, MAX_WAIT_SECONDS); - assertEquals("Task " + asyncId + " not found in completed tasks.", - "found [" + asyncId + "] in completed tasks", message); - } - - @Override - protected void invokeMigrateApi(String sourceCollection, String splitKey, String targetCollection) throws SolrServerException, IOException { - ModifiableSolrParams params = new ModifiableSolrParams(); - String asyncId = "20140128"; - params.set(CollectionParams.ACTION, CollectionParams.CollectionAction.MIGRATE.toString()); - params.set("collection", sourceCollection); - params.set("target.collection", targetCollection); - params.set("split.key", splitKey); - params.set("forward.timeout", 45); - params.set(CommonAdminParams.ASYNC, asyncId); - - invoke(params); - - checkAsyncRequestForCompletion(asyncId); - } - - /** - * Helper method to send a status request with specific retry limit and return - * the message/null from the success response. - */ - private String sendStatusRequestWithRetry(ModifiableSolrParams params, int maxCounter) - throws SolrServerException, IOException { - NamedList status = null; - RequestStatusState state = null; - String message = null; - NamedList r; - while (maxCounter-- > 0) { - r = sendRequest(params); - status = (NamedList) r.get("status"); - state = RequestStatusState.fromKey((String) status.get("state")); - message = (String) status.get("msg"); - - if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED) { - return (String) status.get("msg"); - } - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - - } - - } - // Return last state? - return message; - } - - protected NamedList sendRequest(ModifiableSolrParams params) throws SolrServerException, IOException { - final SolrRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - - String baseUrl = ((HttpSolrClient) shardToJetty.get(SHARD1).get(0).client.solrClient).getBaseURL(); - baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length()); - - try (HttpSolrClient baseServer = getHttpSolrClient(baseUrl)) { - baseServer.setConnectionTimeout(15000); - return baseServer.request(request); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java index 4f92e28dd0c..e886bb67e8d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionReloadTest.java @@ -19,13 +19,11 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; import java.util.concurrent.TimeUnit; -import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.RetryUtil; +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,68 +31,53 @@ import org.slf4j.LoggerFactory; /** * Verifies cluster state remains consistent after collection reload. */ -@Slow @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class CollectionReloadTest extends AbstractFullDistribZkTestBase { +public class CollectionReloadTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public CollectionReloadTest() { - super(); - sliceCount = 1; + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1) + .addConfig("conf", configset("cloud-minimal")) + .configure(); } @Test public void testReloadedLeaderStateAfterZkSessionLoss() throws Exception { - waitForThingsToLevelOut(30000); log.info("testReloadedLeaderStateAfterZkSessionLoss initialized OK ... running test logic"); - String testCollectionName = "c8n_1x1"; - String shardId = "shard1"; - createCollectionRetry(testCollectionName, 1, 1, 1); - cloudClient.setDefaultCollection(testCollectionName); + final String testCollectionName = "c8n_1x1"; + CollectionAdminRequest.createCollection(testCollectionName, "conf", 1, 1) + .process(cluster.getSolrClient()); - Replica leader = getShardLeader(testCollectionName, shardId, 30 /* timeout secs */); + Replica leader + = cluster.getSolrClient().getZkStateReader().getLeaderRetry(testCollectionName, "shard1", DEFAULT_TIMEOUT); - // reload collection and wait to see the core report it has been reloaded - boolean wasReloaded = reloadCollection(leader, testCollectionName); - assertTrue("Collection '"+testCollectionName+"' failed to reload within a reasonable amount of time!", - wasReloaded); + long coreStartTime = getCoreStatus(leader).getCoreStartTime().getTime(); + CollectionAdminRequest.reloadCollection(testCollectionName).process(cluster.getSolrClient()); - // cause session loss - chaosMonkey.expireSession(getJettyOnPort(getReplicaPort(leader))); + RetryUtil.retryUntil("Timed out waiting for core to reload", 30, 1000, TimeUnit.MILLISECONDS, () -> { + long restartTime = 0; + try { + restartTime = getCoreStatus(leader).getCoreStartTime().getTime(); + } catch (Exception e) { + log.warn("Exception getting core start time: {}", e.getMessage()); + return false; + } + return restartTime > coreStartTime; + }); - // TODO: have to wait a while for the node to get marked down after ZK session loss - // but tests shouldn't be so timing dependent! - Thread.sleep(15000); + final int initialStateVersion = getCollectionState(testCollectionName).getZNodeVersion(); - // wait up to 15 seconds to see the replica in the active state - String replicaState = null; - int timeoutSecs = 15; - long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutSecs, TimeUnit.SECONDS); - while (System.nanoTime() < timeout) { - // state of leader should be active after session loss recovery - see SOLR-7338 - cloudClient.getZkStateReader().forceUpdateCollection(testCollectionName); - ClusterState cs = cloudClient.getZkStateReader().getClusterState(); - Slice slice = cs.getSlice(testCollectionName, shardId); - replicaState = slice.getReplica(leader.getName()).getStr(ZkStateReader.STATE_PROP); - if ("active".equals(replicaState)) - break; + cluster.expireZkSession(cluster.getReplicaJetty(leader)); - Thread.sleep(1000); - } - assertEquals("Leader state should be active after recovering from ZK session loss, but after " + - timeoutSecs + " seconds, it is " + replicaState, "active", replicaState); - - // try to clean up - try { - new CollectionAdminRequest.Delete() - .setCollectionName(testCollectionName).process(cloudClient); - } catch (Exception e) { - // don't fail the test - log.warn("Could not delete collection {} after test completed", testCollectionName); - } + waitForState("Timed out waiting for core to re-register as ACTIVE after session expiry", testCollectionName, (n, c) -> { + log.info("Collection state: {}", c.toString()); + Replica expiredReplica = c.getReplica(leader.getName()); + return expiredReplica.getState() == Replica.State.ACTIVE && c.getZNodeVersion() > initialStateVersion; + }); log.info("testReloadedLeaderStateAfterZkSessionLoss succeeded ... shutting down now!"); } diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionStateFormat2Test.java b/solr/core/src/test/org/apache/solr/cloud/CollectionStateFormat2Test.java index 9b223651d36..91eb4617476 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionStateFormat2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionStateFormat2Test.java @@ -16,74 +16,55 @@ */ package org.apache.solr.cloud; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.zookeeper.data.Stat; +import org.junit.BeforeClass; import org.junit.Test; -public class CollectionStateFormat2Test extends AbstractFullDistribZkTestBase { +public class CollectionStateFormat2Test extends SolrCloudTestCase { - protected String getSolrXml() { - return "solr.xml"; + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(4) + .addConfig("conf", configset("cloud-minimal")) + .configure(); } @Test - @ShardsFixed(num = 4) - public void test() throws Exception { - try (CloudSolrClient client = createCloudClient(null)) { - testZkNodeLocation(client); - testConfNameAndCollectionNameSame(client); - } - } + public void testConfNameAndCollectionNameSame() throws Exception { - @Override - protected String getStateFormat() { - return "2"; - } - - private void testConfNameAndCollectionNameSame(CloudSolrClient client) throws Exception{ // .system collection precreates the configset - - createCollection(".system", client, 2, 1); - waitForRecoveriesToFinish(".system", false); + CollectionAdminRequest.createCollection(".system", 2, 1) + .process(cluster.getSolrClient()); } - private void testZkNodeLocation(CloudSolrClient client) throws Exception{ + @Test + public void testZkNodeLocation() throws Exception { String collectionName = "myExternColl"; + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) + .process(cluster.getSolrClient()); - createCollection(collectionName, client, 2, 2); + waitForState("Collection not created", collectionName, (n, c) -> DocCollection.isFullyActive(n, c, 2, 2)); + assertTrue("State Format 2 collection path does not exist", + zkClient().exists(ZkStateReader.getCollectionPath(collectionName), true)); - waitForRecoveriesToFinish(collectionName, false); - assertTrue("does not exist collection state externally", - cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.getCollectionPath(collectionName), true)); Stat stat = new Stat(); - byte[] data = cloudClient.getZkStateReader().getZkClient().getData(ZkStateReader.getCollectionPath(collectionName), null, stat, true); - DocCollection c = ZkStateReader.getCollectionLive(cloudClient.getZkStateReader(), collectionName); - ClusterState clusterState = cloudClient.getZkStateReader().getClusterState(); - assertEquals("The zkversion of the nodes must be same zkver:" + stat.getVersion() , stat.getVersion(),clusterState.getCollection(collectionName).getZNodeVersion() ); - assertTrue("DocCllection#getStateFormat() must be > 1", cloudClient.getZkStateReader().getClusterState().getCollection(collectionName).getStateFormat() > 1); + zkClient().getData(ZkStateReader.getCollectionPath(collectionName), null, stat, true); + DocCollection c = getCollectionState(collectionName); + + assertEquals("DocCollection version should equal the znode version", stat.getVersion(), c.getZNodeVersion() ); + assertTrue("DocCollection#getStateFormat() must be > 1", c.getStateFormat() > 1); // remove collection - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionParams.CollectionAction.DELETE.toString()); - params.set("name", collectionName); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - if (client == null) { - client = createCloudClient(null); - } + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + waitForState("Collection not deleted", collectionName, (n, coll) -> coll == null); - client.request(request); - - assertCollectionNotExists(collectionName, 45); - assertFalse("collection state should not exist externally", cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.getCollectionPath(collectionName), true)); + assertFalse("collection state should not exist externally", + zkClient().exists(ZkStateReader.getCollectionPath(collectionName), true)); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java index fadb880b8b6..58c956b6ad0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MigrateRouteKeyTest.java @@ -16,65 +16,53 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Map; +import java.util.concurrent.TimeUnit; + import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.RoutingRule; import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.Utils; import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; +public class MigrateRouteKeyTest extends SolrCloudTestCase { -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; -import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(2) + .addConfig("conf", configset("cloud-minimal")) + .configure(); -public class MigrateRouteKeyTest extends BasicDistributedZkTest { - - public MigrateRouteKeyTest() { - schemaString = "schema15.xml"; // we need a string id + if (usually()) { + CollectionAdminRequest.setClusterProperty("legacyCloud", "false").process(cluster.getSolrClient()); + } } private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @Test - public void test() throws Exception { - waitForThingsToLevelOut(15); - - if (usually()) { - log.info("Using legacyCloud=false for cluster"); - CollectionsAPIDistributedZkTest.setClusterProp(cloudClient, "legacyCloud", "false"); - } - multipleShardMigrateTest(); - printLayout(); - } - - private boolean waitForRuleToExpire(String splitKey, long finishTime) throws KeeperException, InterruptedException, SolrServerException, IOException { - ClusterState state;Slice slice; + private boolean waitForRuleToExpire(String collection, String shard, String splitKey, long finishTime) throws KeeperException, InterruptedException, SolrServerException, IOException { + DocCollection state; + Slice slice; boolean ruleRemoved = false; long expiryTime = finishTime + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS); while (System.nanoTime() < expiryTime) { - getCommonCloudSolrClient().getZkStateReader().forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); - state = getCommonCloudSolrClient().getZkStateReader().getClusterState(); - slice = state.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD2); + cluster.getSolrClient().getZkStateReader().forceUpdateCollection(collection); + state = getCollectionState(collection); + slice = state.getSlice(shard); Map routingRules = slice.getRoutingRules(); if (routingRules == null || routingRules.isEmpty() || !routingRules.containsKey(splitKey)) { ruleRemoved = true; @@ -82,59 +70,28 @@ public class MigrateRouteKeyTest extends BasicDistributedZkTest { } SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", splitKey + random().nextInt()); - cloudClient.add(doc); + cluster.getSolrClient().add(collection, doc); Thread.sleep(1000); } return ruleRemoved; } - protected void invokeMigrateApi(String sourceCollection, String splitKey, String targetCollection) throws SolrServerException, IOException { - cloudClient.setDefaultCollection(sourceCollection); - CollectionAdminRequest.Migrate migrateRequest = new CollectionAdminRequest.Migrate(); - migrateRequest.setCollectionName(sourceCollection); - migrateRequest.setTargetCollection(targetCollection); - migrateRequest.setSplitKey(splitKey); - migrateRequest.setForwardTimeout(45); - migrateRequest.process(cloudClient); - } - - protected void invoke(ModifiableSolrParams params) throws SolrServerException, IOException { - SolrRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - - String baseUrl = ((HttpSolrClient) shardToJetty.get(SHARD1).get(0).client.solrClient) - .getBaseURL(); - baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length()); - - try (HttpSolrClient baseClient = getHttpSolrClient(baseUrl)) { - baseClient.setConnectionTimeout(15000); - baseClient.setSoTimeout(60000 * 5); - baseClient.request(request); + protected void invokeCollectionMigration(CollectionAdminRequest.AsyncCollectionAdminRequest request) throws IOException, SolrServerException, InterruptedException { + if (random().nextBoolean()) { + cluster.getSolrClient().setSoTimeout(60000); // can take a while + request.process(cluster.getSolrClient()); + } + else { + request.processAndWait(cluster.getSolrClient(), 60000); } } - private void createCollection(String targetCollection) throws Exception { - HashMap> collectionInfos = new HashMap<>(); + @Test + public void multipleShardMigrateTest() throws Exception { - try (CloudSolrClient client = createCloudClient(null)) { - Map props = Utils.makeMap( - REPLICATION_FACTOR, 1, - MAX_SHARDS_PER_NODE, 5, - NUM_SLICES, 1); + CollectionAdminRequest.createCollection("sourceCollection", "conf", 2, 1).process(cluster.getSolrClient()); + cluster.getSolrClient().setDefaultCollection("sourceCollection"); - createCollection(collectionInfos, targetCollection, props, client); - } - - List list = collectionInfos.get(targetCollection); - checkForCollection(targetCollection, list, null); - - waitForRecoveriesToFinish(targetCollection, false); - } - - protected void multipleShardMigrateTest() throws Exception { - del("*:*"); - commit(); - assertTrue(cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound() == 0); final String splitKey = "a"; final int BIT_SEP = 1; final int[] splitKeyCount = new int[1]; @@ -147,38 +104,41 @@ public class MigrateRouteKeyTest extends BasicDistributedZkTest { SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", key + "!" + id); doc.addField("n_ti", id); - cloudClient.add(doc); + cluster.getSolrClient().add("sourceCollection", doc); if (splitKey.equals(shardKey)) splitKeyCount[0]++; } assertTrue(splitKeyCount[0] > 0); String targetCollection = "migrate_multipleshardtest_targetCollection"; - createCollection(targetCollection); + CollectionAdminRequest.createCollection(targetCollection, "conf", 1, 1).process(cluster.getSolrClient()); - Indexer indexer = new Indexer(cloudClient, splitKey, 1, 30); + Indexer indexer = new Indexer(cluster.getSolrClient(), splitKey, 1, 30); indexer.start(); - String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), targetCollection); - - try (HttpSolrClient collectionClient = getHttpSolrClient(url)) { + DocCollection state = getCollectionState(targetCollection); + Replica replica = state.getReplicas().get(0); + try (HttpSolrClient collectionClient = getHttpSolrClient(replica.getCoreUrl())) { SolrQuery solrQuery = new SolrQuery("*:*"); assertEquals("DocCount on target collection does not match", 0, collectionClient.query(solrQuery).getResults().getNumFound()); - invokeMigrateApi(AbstractDistribZkTestBase.DEFAULT_COLLECTION, splitKey + "/" + BIT_SEP + "!", targetCollection); + invokeCollectionMigration( + CollectionAdminRequest.migrateData("sourceCollection", targetCollection, splitKey + "/" + BIT_SEP + "!") + .setForwardTimeout(45)); + long finishTime = System.nanoTime(); indexer.join(); splitKeyCount[0] += indexer.getSplitKeyCount(); try { - cloudClient.deleteById("a/" + BIT_SEP + "!104"); + cluster.getSolrClient().deleteById("a/" + BIT_SEP + "!104"); splitKeyCount[0]--; } catch (Exception e) { log.warn("Error deleting document a/" + BIT_SEP + "!104", e); } - cloudClient.commit(); + cluster.getSolrClient().commit(); collectionClient.commit(); solrQuery = new SolrQuery("*:*").setRows(1000); @@ -186,14 +146,20 @@ public class MigrateRouteKeyTest extends BasicDistributedZkTest { log.info("Response from target collection: " + response); assertEquals("DocCount on target collection does not match", splitKeyCount[0], response.getResults().getNumFound()); - getCommonCloudSolrClient().getZkStateReader().forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); - ClusterState state = getCommonCloudSolrClient().getZkStateReader().getClusterState(); - Slice slice = state.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD2); - assertNotNull("Routing rule map is null", slice.getRoutingRules()); - assertFalse("Routing rule map is empty", slice.getRoutingRules().isEmpty()); - assertNotNull("No routing rule exists for route key: " + splitKey, slice.getRoutingRules().get(splitKey + "!")); + waitForState("Expected to find routing rule for split key " + splitKey, "sourceCollection", (n, c) -> { + if (c == null) + return false; + Slice shard = c.getSlice("shard2"); + if (shard == null) + return false; + if (shard.getRoutingRules() == null || shard.getRoutingRules().isEmpty()) + return false; + if (shard.getRoutingRules().get(splitKey + "!") == null) + return false; + return true; + }); - boolean ruleRemoved = waitForRuleToExpire(splitKey, finishTime); + boolean ruleRemoved = waitForRuleToExpire("sourceCollection", "shard2", splitKey, finishTime); assertTrue("Routing rule was not expired", ruleRemoved); } } @@ -230,7 +196,7 @@ public class MigrateRouteKeyTest extends BasicDistributedZkTest { try { Thread.sleep(50); } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + return; } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java index 5cb329aac53..1b74cbe7804 100644 --- a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java @@ -17,23 +17,19 @@ package org.apache.solr.cloud.rule; import java.lang.invoke.MethodHandles; - import java.nio.file.Paths; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.solr.client.solrj.SolrClient; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.cloud.AbstractFullDistribZkTestBase; +import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.ImplicitDocRouter; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ModifiableSolrParams; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.rules.ExpectedException; import org.slf4j.Logger; @@ -43,34 +39,35 @@ import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH; import static org.junit.matchers.JUnitMatchers.containsString; -public class RulesTest extends AbstractFullDistribZkTestBase { +@LuceneTestCase.Slow +public class RulesTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(5) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } + @org.junit.Rule public ExpectedException expectedException = ExpectedException.none(); @Test - @ShardsFixed(num = 5) public void doIntegrationTest() throws Exception { final long minGB = (random().nextBoolean() ? 1 : 0); - assumeTrue("doIntegrationTest needs minGB="+minGB+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB); + assumeTrue("doIntegrationTest needs minGB="+minGB+" usable disk space", + ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB); + String rulesColl = "rulesColl"; - try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) { - CollectionAdminResponse rsp; - CollectionAdminRequest.Create create = new CollectionAdminRequest.Create() - .setCollectionName(rulesColl) - .setShards("shard1") - .setRouterName(ImplicitDocRouter.NAME) - .setReplicationFactor(2) - .setRule("cores:<4", "node:*,replica:<2", "freedisk:>"+minGB) - .setSnitch("class:ImplicitSnitch"); - rsp = create.process(client); - assertEquals(0, rsp.getStatus()); - assertTrue(rsp.isSuccess()); + CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2) + .setRule("cores:<4", "node:*,replica:<2", "freedisk:>"+minGB) + .setSnitch("class:ImplicitSnitch") + .process(cluster.getSolrClient()); - } + DocCollection rulesCollection = getCollectionState(rulesColl); - DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl); List list = (List) rulesCollection.get("rule"); assertEquals(3, list.size()); assertEquals ( "<4", ((Map)list.get(0)).get("cores")); @@ -80,51 +77,25 @@ public class RulesTest extends AbstractFullDistribZkTestBase { assertEquals(1, list.size()); assertEquals ( "ImplicitSnitch", ((Map)list.get(0)).get("class")); - try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) { - CollectionAdminResponse rsp; - CollectionAdminRequest.CreateShard createShard = new CollectionAdminRequest.CreateShard() - .setCollectionName(rulesColl) - .setShardName("shard2"); - rsp = createShard.process(client); - assertEquals(0, rsp.getStatus()); - assertTrue(rsp.isSuccess()); - - CollectionAdminRequest.AddReplica addReplica = new CollectionAdminRequest.AddReplica() - .setCollectionName(rulesColl) - .setShardName("shard2"); - rsp = addReplica.process(client); - assertEquals(0, rsp.getStatus()); - assertTrue(rsp.isSuccess()); - } - + CollectionAdminRequest.createShard(rulesColl, "shard2").process(cluster.getSolrClient()); + CollectionAdminRequest.addReplicaToShard(rulesColl, "shard2").process(cluster.getSolrClient()); } @Test public void testPortRule() throws Exception { + + JettySolrRunner jetty = cluster.getRandomJetty(random()); + String port = Integer.toString(jetty.getLocalPort()); + String rulesColl = "portRuleColl"; - String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0)); - String port = "-1"; - Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl); - if (hostAndPortMatcher.find()) { - port = hostAndPortMatcher.group(2); - } - try (SolrClient client = createNewSolrClient("", baseUrl)) { - CollectionAdminResponse rsp; - CollectionAdminRequest.Create create = new CollectionAdminRequest.Create(); - create.setCollectionName(rulesColl); - create.setShards("shard1"); - create.setRouterName(ImplicitDocRouter.NAME); - create.setReplicationFactor(2); - create.setRule("port:" + port); - create.setSnitch("class:ImplicitSnitch"); - rsp = create.process(client); - assertEquals(0, rsp.getStatus()); - assertTrue(rsp.isSuccess()); + CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2) + .setRule("port:" + port) + .setSnitch("class:ImplicitSnitch") + .process(cluster.getSolrClient()); - } + DocCollection rulesCollection = getCollectionState(rulesColl); - DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl); List list = (List) rulesCollection.get("rule"); assertEquals(1, list.size()); assertEquals(port, ((Map) list.get(0)).get("port")); @@ -135,33 +106,21 @@ public class RulesTest extends AbstractFullDistribZkTestBase { @Test public void testHostFragmentRule() throws Exception { - String rulesColl = "ipRuleColl"; - String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0)); - String ip_1 = "-1"; - String ip_2 = "-1"; - Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl); - if (hostAndPortMatcher.find()) { - String[] ipFragments = hostAndPortMatcher.group(1).split("\\."); - ip_1 = ipFragments[ipFragments.length - 1]; - ip_2 = ipFragments[ipFragments.length - 2]; - } - try (SolrClient client = createNewSolrClient("", baseUrl)) { - CollectionAdminResponse rsp; - CollectionAdminRequest.Create create = new CollectionAdminRequest.Create(); - create.setCollectionName(rulesColl); - create.setShards("shard1"); - create.setRouterName(ImplicitDocRouter.NAME); - create.setReplicationFactor(2); - create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1); - create.setSnitch("class:ImplicitSnitch"); - rsp = create.process(client); - assertEquals(0, rsp.getStatus()); - assertTrue(rsp.isSuccess()); + String rulesColl = "hostFragment"; - } + JettySolrRunner jetty = cluster.getRandomJetty(random()); + String host = jetty.getBaseUrl().getHost(); + String[] ipFragments = host.split("\\."); + String ip_1 = ipFragments[ipFragments.length - 1]; + String ip_2 = ipFragments[ipFragments.length - 2]; - DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl); + CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2) + .setRule("ip_2:" + ip_2, "ip_1:" + ip_1) + .setSnitch("class:ImplicitSnitch") + .process(cluster.getSolrClient()); + + DocCollection rulesCollection = getCollectionState(rulesColl); List list = (List) rulesCollection.get("rule"); assertEquals(2, list.size()); assertEquals(ip_2, list.get(0).get("ip_2")); @@ -175,66 +134,54 @@ public class RulesTest extends AbstractFullDistribZkTestBase { @Test public void testHostFragmentRuleThrowsExceptionWhenIpDoesNotMatch() throws Exception { + String rulesColl = "ipRuleColl"; - String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0)); - String ip_1 = "-1"; - String ip_2 = "-1"; - Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl); - if (hostAndPortMatcher.find()) { - String[] ipFragments = hostAndPortMatcher.group(1).split("\\."); - ip_1 = ipFragments[ipFragments.length - 1]; - ip_2 = ipFragments[ipFragments.length - 2]; - } - try (SolrClient client = createNewSolrClient("", baseUrl)) { - CollectionAdminRequest.Create create = new CollectionAdminRequest.Create(); - create.setCollectionName(rulesColl); - create.setShards("shard1"); - create.setRouterName(ImplicitDocRouter.NAME); - create.setReplicationFactor(2); + JettySolrRunner jetty = cluster.getRandomJetty(random()); + String host = jetty.getBaseUrl().getHost(); + String[] ipFragments = host.split("\\."); + String ip_1 = ipFragments[ipFragments.length - 1]; + String ip_2 = ipFragments[ipFragments.length - 2]; - create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1 + "9999"); - create.setSnitch("class:ImplicitSnitch"); + expectedException.expect(HttpSolrClient.RemoteSolrException.class); + expectedException.expectMessage(containsString("ip_1")); - expectedException.expect(HttpSolrClient.RemoteSolrException.class); - expectedException.expectMessage(containsString("ip_1")); - - create.process(client); - } + CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2) + .setRule("ip_2:" + ip_2, "ip_1:" + ip_1 + "9999") + .setSnitch("class:ImplicitSnitch") + .process(cluster.getSolrClient()); } @Test public void testModifyColl() throws Exception { + final long minGB1 = (random().nextBoolean() ? 1 : 0); final long minGB2 = 5; - assumeTrue("testModifyColl needs minGB1="+minGB1+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB1); - assumeTrue("testModifyColl needs minGB2="+minGB2+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB2); + assumeTrue("testModifyColl needs minGB1="+minGB1+" usable disk space", + ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB1); + assumeTrue("testModifyColl needs minGB2="+minGB2+" usable disk space", + ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB2); + String rulesColl = "modifyColl"; - try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) { - CollectionAdminResponse rsp; - CollectionAdminRequest.Create create = new CollectionAdminRequest.Create() - .setCollectionName(rulesColl) - .setNumShards(1) - .setReplicationFactor(2) - .setRule("cores:<4", "node:*,replica:1", "freedisk:>"+minGB1) - .setSnitch("class:ImplicitSnitch"); - rsp = create.process(client); - assertEquals(0, rsp.getStatus()); - assertTrue(rsp.isSuccess()); - ModifiableSolrParams p = new ModifiableSolrParams(); - p.add("collection", rulesColl); - p.add("action", "MODIFYCOLLECTION"); - p.add("rule", "cores:<5"); - p.add("rule", "node:*,replica:1"); - p.add("rule", "freedisk:>"+minGB2); - p.add("autoAddReplicas", "true"); - client.request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p)); - } + CollectionAdminRequest.createCollection(rulesColl, "conf", 1, 2) + .setRule("cores:<4", "node:*,replica:1", "freedisk:>" + minGB1) + .setSnitch("class:ImplicitSnitch") + .process(cluster.getSolrClient()); - DocCollection rulesCollection = ZkStateReader.getCollectionLive(cloudClient.getZkStateReader(), rulesColl); + // TODO: Make a MODIFYCOLLECTION SolrJ class + ModifiableSolrParams p = new ModifiableSolrParams(); + p.add("collection", rulesColl); + p.add("action", "MODIFYCOLLECTION"); + p.add("rule", "cores:<5"); + p.add("rule", "node:*,replica:1"); + p.add("rule", "freedisk:>"+minGB2); + p.add("autoAddReplicas", "true"); + cluster.getSolrClient().request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p)); + + DocCollection rulesCollection = getCollectionState(rulesColl); log.info("version_of_coll {} ", rulesCollection.getZNodeVersion()); List list = (List) rulesCollection.get("rule"); assertEquals(3, list.size()); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index e57bcf90946..af5d74b72ea 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -32,6 +32,7 @@ import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.ImplicitDocRouter; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CollectionParams.CollectionAction; @@ -319,6 +320,31 @@ public abstract class CollectionAdminRequest return new Create(collection, config, numShards, numReplicas); } + /** + * Returns a SolrRequest for creating a collection using a default configSet + * + * This requires that there is either a single configset configured in the cluster, or + * that there is a configset with the same name as the collection + * + * @param collection the collection name + * @param numShards the number of shards in the collection + * @param numReplicas the replication factor of the collection + */ + public static Create createCollection(String collection, int numShards, int numReplicas) { + return new Create(collection, numShards, numReplicas); + } + + /** + * Returns a SolrRequest for creating a collection with the implicit router + * @param collection the collection name + * @param config the collection config + * @param shards a shard definition string + * @param numReplicas the replication factor of the collection + */ + public static Create createCollectionWithImplicitRouter(String collection, String config, String shards, int numReplicas) { + return new Create(collection, config, shards, numReplicas); + } + // CREATE request public static class Create extends AsyncCollectionSpecificAdminRequest { @@ -351,6 +377,20 @@ public abstract class CollectionAdminRequest this.replicationFactor = numReplicas; } + private Create(String collection, int numShards, int numReplicas) { + super(CollectionAction.CREATE, SolrIdentifierValidator.validateCollectionName(collection)); + this.numShards = numShards; + this.replicationFactor = numReplicas; + } + + private Create(String collection, String config, String shards, int numReplicas) { + super(CollectionAction.CREATE, SolrIdentifierValidator.validateCollectionName(collection)); + this.configName = config; + this.replicationFactor = numReplicas; + this.shards = shards; + this.routerName = ImplicitDocRouter.NAME; + } + @Deprecated public Create setConfigName(String config) { this.configName = config; return this; } public Create setCreateNodeSet(String nodeSet) { this.createNodeSet = nodeSet; return this; } @@ -424,16 +464,20 @@ public abstract class CollectionAdminRequest public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); - params.set("collection.configName", configName); - params.set("createNodeSet", createNodeSet); + if (configName != null) + params.set("collection.configName", configName); + if (createNodeSet != null) + params.set("createNodeSet", createNodeSet); if (numShards != null) { params.set( ZkStateReader.NUM_SHARDS_PROP, numShards); } if (maxShardsPerNode != null) { params.set( "maxShardsPerNode", maxShardsPerNode); } - params.set( "router.name", routerName); - params.set("shards", shards); + if (routerName != null) + params.set( "router.name", routerName); + if (shards != null) + params.set("shards", shards); if (routerField != null) { params.set("router.field", routerField); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreStatus.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreStatus.java index e1e387c2846..3952b28f300 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreStatus.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreStatus.java @@ -17,6 +17,8 @@ package org.apache.solr.client.solrj.request; +import java.util.Date; + import org.apache.solr.common.util.NamedList; public class CoreStatus { @@ -39,4 +41,8 @@ public class CoreStatus { public String toString() { return response.toString(); } + + public Date getCoreStartTime() { + return (Date) response.get("startTime"); + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/RetryUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/RetryUtil.java index 2b9a0fd231e..a21074159c5 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/RetryUtil.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/RetryUtil.java @@ -19,7 +19,6 @@ package org.apache.solr.common.util; import java.lang.invoke.MethodHandles; import java.util.Collections; import java.util.Set; - import java.util.concurrent.TimeUnit; import org.apache.solr.common.SolrException; @@ -68,6 +67,16 @@ public class RetryUtil { } return false; } + + public static void retryUntil(String errorMessage, int retries, long pauseTime, TimeUnit pauseUnit, BooleanRetryCmd cmd) + throws InterruptedException { + while (retries-- > 0) { + if (cmd.execute()) + return; + pauseUnit.sleep(pauseTime); + } + throw new SolrException(ErrorCode.SERVER_ERROR, errorMessage); + } public static void retryOnBoolean(long timeoutms, long intervalms, BooleanRetryCmd cmd) { long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutms, TimeUnit.MILLISECONDS); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index 910a18aeda3..74cae539e60 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -19,6 +19,7 @@ package org.apache.solr.cloud; import javax.servlet.Filter; import java.io.File; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -57,13 +58,18 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrjNamedThreadFactory; +import org.apache.solr.core.CoreContainer; import org.apache.zookeeper.KeeperException; import org.eclipse.jetty.servlet.ServletHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * "Mini" SolrCloud cluster to be used for testing */ public class MiniSolrCloudCluster { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String DEFAULT_CLOUD_SOLR_XML = "\n" + "\n" + @@ -187,6 +193,8 @@ public class MiniSolrCloudCluster { this.baseDir = Objects.requireNonNull(baseDir); this.jettyConfig = Objects.requireNonNull(jettyConfig); + log.info("Starting cluster of {} servers in {}", numServers, baseDir); + Files.createDirectories(baseDir); this.externalZkServer = zkTestServer != null; @@ -513,4 +521,18 @@ public class MiniSolrCloudCluster { } throw new IllegalArgumentException("Cannot find Jetty for a replica with core url " + replica.getCoreUrl()); } + + /** + * Make the zookeeper session on a particular jetty expire + */ + public void expireZkSession(JettySolrRunner jetty) { + CoreContainer cores = jetty.getCoreContainer(); + if (cores != null) { + SolrZkClient zkClient = cores.getZkController().getZkClient(); + zkClient.getSolrZooKeeper().closeCnxn(); + long sessionId = zkClient.getSolrZooKeeper().getSessionId(); + zkServer.expire(sessionId); + log.info("Expired zookeeper session {} from node {}", sessionId, jetty.getBaseUrl()); + } + } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java index 02a4895ac99..b64b1ce1a42 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java @@ -229,7 +229,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 { return predicate.matches(n, c); }); } catch (Exception e) { - fail(message + "\nLast available state: " + state.get()); + fail(message + "\n" + e.getMessage() + "\nLast available state: " + state.get()); } } @@ -239,6 +239,8 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 { */ public static CollectionStatePredicate clusterShape(int expectedShards, int expectedReplicas) { return (liveNodes, collectionState) -> { + if (collectionState == null) + return false; if (collectionState.getSlices().size() != expectedShards) return false; for (Slice slice : collectionState) { @@ -257,7 +259,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 { /** * Get a (reproducibly) random shard from a {@link DocCollection} */ - protected Slice getRandomShard(DocCollection collection) { + protected static Slice getRandomShard(DocCollection collection) { List shards = new ArrayList<>(collection.getActiveSlices()); if (shards.size() == 0) fail("Couldn't get random shard for collection as it has no shards!\n" + collection.toString()); @@ -268,7 +270,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 { /** * Get a (reproducibly) random replica from a {@link Slice} */ - protected Replica getRandomReplica(Slice slice) { + protected static Replica getRandomReplica(Slice slice) { List replicas = new ArrayList<>(slice.getReplicas()); if (replicas.size() == 0) fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString()); @@ -279,7 +281,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 { /** * Get a (reproducibly) random replica from a {@link Slice} matching a predicate */ - protected Replica getRandomReplica(Slice slice, Predicate matchPredicate) { + protected static Replica getRandomReplica(Slice slice, Predicate matchPredicate) { List replicas = new ArrayList<>(slice.getReplicas()); if (replicas.size() == 0) fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString()); @@ -297,7 +299,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 { * * This assumes that the replica is hosted on a live node. */ - protected CoreStatus getCoreStatus(Replica replica) throws IOException, SolrServerException { + protected static CoreStatus getCoreStatus(Replica replica) throws IOException, SolrServerException { JettySolrRunner jetty = cluster.getReplicaJetty(replica); try (HttpSolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString(), cluster.getSolrClient().getHttpClient())) { return CoreAdminRequest.getCoreStatus(replica.getCoreName(), client);