Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-13438: on collection delete, also delete .AUTOCREATED config set (#1759) 

Co-authored-by: Anderson Dorow <Anderson.Dorow@zooplus.com>
This commit is contained in:
Anderson Dorow 2020-08-21 21:43:16 +02:00 committed by GitHub
parent 37cd17dcf5
commit 66b6ce2cd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 173 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -44,6 +44,8 @@ Improvements
* SOLR-14615: CPU Utilization Based Circuit Breaker (Atri Sharma)
* SOLR-13438: On deleting a collection, its config set will also be deleted iff it has been auto-created, and not used by any other collection (Anderson Dorow)
Other Changes
----------------------
* SOLR-14656: Autoscaling framework removed (Ishan Chattopadhyaya, noble, Ilan Ginzburg)

28
solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
View File

@ -45,6 +45,7 @@ import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.handler.admin.ConfigSetsHandlerApi;
import org.apache.solr.handler.admin.MetricsHistoryHandler;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.zookeeper.KeeperException;
@ -160,6 +161,33 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
});
}
// delete related config set iff: it is auto generated AND not related to any other collection
String configSetName = zkStateReader.readConfigName(collection);
if (ConfigSetsHandlerApi.isAutoGeneratedConfigSet(configSetName)) {
boolean configSetIsUsedByOtherCollection = false;
// make sure the configSet is not shared with other collections
// Similar to what happens in: OverseerConfigSetMessageHandler::deleteConfigSet
for (Map.Entry<String, DocCollection> entry : zkStateReader.getClusterState().getCollectionsMap().entrySet()) {
String otherConfigSetName = null;
try {
otherConfigSetName = zkStateReader.readConfigName(entry.getKey());
} catch (KeeperException ex) {
// ignore 'no config found' errors
}
if (configSetName.equals(otherConfigSetName)) {
configSetIsUsedByOtherCollection = true;
break;
}
}
if (!configSetIsUsedByOtherCollection) {
// delete the config set
zkStateReader.getConfigManager().deleteConfigDir(configSetName);
}
}
// TimeOut timeout = new TimeOut(60, TimeUnit.SECONDS, timeSource);
// boolean removed = false;
// while (! timeout.hasTimedOut()) {

4
solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java
View File

@ -42,6 +42,10 @@ public class ConfigSetsHandlerApi extends BaseHandlerApiSupport {
return configName + AUTOCREATED_CONFIGSET_SUFFIX;
}
public static boolean isAutoGeneratedConfigSet(String configName) {
return configName != null && configName.endsWith(AUTOCREATED_CONFIGSET_SUFFIX);
}
private static Collection<ApiCommand> createMapping() {
Map<ConfigSetMeta, ApiCommand> result = new EnumMap<>(ConfigSetMeta.class);

200
solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.java
View File

@ -16,10 +16,6 @@
*/
package org.apache.solr.cloud.api.collections;
import java.util.Collection;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
@ -32,69 +28,151 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.util.TimeOut;
import org.junit.Test;
import java.util.Collection;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
public class SimpleCollectionCreateDeleteTest extends AbstractFullDistribZkTestBase {
public SimpleCollectionCreateDeleteTest() {
sliceCount = 1;
}
@Test
@ShardsFixed(num = 1)
public void test() throws Exception {
String overseerNode = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient());
String notOverseerNode = null;
for (CloudJettyRunner cloudJetty : cloudJettys) {
if (!overseerNode.equals(cloudJetty.nodeName)) {
notOverseerNode = cloudJetty.nodeName;
break;
}
public SimpleCollectionCreateDeleteTest() {
sliceCount = 1;
}
String collectionName = "SimpleCollectionCreateDeleteTest";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,1,1)
.setCreateNodeSet(overseerNode);
NamedList<Object> request = create.process(cloudClient).getResponse();
if (request.get("success") != null) {
assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
@SuppressWarnings({"rawtypes"})
CollectionAdminRequest delete = CollectionAdminRequest.deleteCollection(collectionName);
cloudClient.request(delete);
assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
// currently, removing a collection does not wait for cores to be unloaded
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
while (true) {
if( timeout.hasTimedOut() ) {
throw new TimeoutException("Timed out waiting for all collections to be fully removed.");
}
boolean allContainersEmpty = true;
for(JettySolrRunner jetty : jettys) {
Collection<SolrCore> cores = jetty.getCoreContainer().getCores();
for (SolrCore core : cores) {
CoreDescriptor cd = core.getCoreDescriptor();
if (cd != null) {
if (cd.getCloudDescriptor().getCollectionName().equals(collectionName)) {
allContainersEmpty = false;
}
@Test
@ShardsFixed(num = 1)
public void testCreateAndDeleteThenCreateAgain() throws Exception {
String overseerNode = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient());
String notOverseerNode = null;
for (CloudJettyRunner cloudJetty : cloudJettys) {
if (!overseerNode.equals(cloudJetty.nodeName)) {
notOverseerNode = cloudJetty.nodeName;
break;
}
}
}
if (allContainersEmpty) {
break;
}
}
String collectionName = "SimpleCollectionCreateDeleteTest";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, 1, 1)
.setCreateNodeSet(overseerNode);
// create collection again on a node other than the overseer leader
create = CollectionAdminRequest.createCollection(collectionName,1,1)
.setCreateNodeSet(notOverseerNode);
request = create.process(cloudClient).getResponse();
assertTrue("Collection creation should not have failed", request.get("success") != null);
NamedList<Object> request = create.process(cloudClient).getResponse();
if (request.get("success") != null) {
assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(collectionName);
cloudClient.request(delete);
assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
// currently, removing a collection does not wait for cores to be unloaded
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
while (true) {
if (timeout.hasTimedOut()) {
throw new TimeoutException("Timed out waiting for all collections to be fully removed.");
}
boolean allContainersEmpty = true;
for (JettySolrRunner jetty : jettys) {
Collection<SolrCore> cores = jetty.getCoreContainer().getCores();
for (SolrCore core : cores) {
CoreDescriptor cd = core.getCoreDescriptor();
if (cd != null) {
if (cd.getCloudDescriptor().getCollectionName().equals(collectionName)) {
allContainersEmpty = false;
}
}
}
}
if (allContainersEmpty) {
break;
}
}
// create collection again on a node other than the overseer leader
create = CollectionAdminRequest.createCollection(collectionName, 1, 1)
.setCreateNodeSet(notOverseerNode);
request = create.process(cloudClient).getResponse();
assertTrue("Collection creation should not have failed", request.get("success") != null);
}
}
}
@Test
@ShardsFixed(num = 1)
public void testDeleteAlsoDeletesAutocreatedConfigSet() throws Exception {
String collectionName = "SimpleCollectionCreateDeleteTest.testDeleteAlsoDeletesAutocreatedConfigSet";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, 1, 1);
NamedList<Object> request = create.process(cloudClient).getResponse();
if (request.get("success") != null) {
// collection exists now
assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
String configName = cloudClient.getZkStateReader().readConfigName(collectionName);
// config for this collection is '.AUTOCREATED', and exists globally
assertTrue(configName.endsWith(".AUTOCREATED"));
assertTrue(cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(collectionName);
cloudClient.request(delete);
// collection has been deleted
assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
// ... and so has its autocreated config set
assertFalse("The auto-created config set should have been deleted with its collection", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
}
}
@Test
@ShardsFixed(num = 1)
public void testDeleteDoesNotDeleteSharedAutocreatedConfigSet() throws Exception {
String collectionNameInitial = "SimpleCollectionCreateDeleteTest.initialCollection";
CollectionAdminRequest.Create createInitial = CollectionAdminRequest.createCollection(collectionNameInitial, 1, 1);
NamedList<Object> requestInitial = createInitial.process(cloudClient).getResponse();
if (requestInitial.get("success") != null) {
// collection exists now
assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameInitial, false));
String configName = cloudClient.getZkStateReader().readConfigName(collectionNameInitial);
// config for this collection is '.AUTOCREATED', and exists globally
assertTrue(configName.endsWith(".AUTOCREATED"));
assertTrue(cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
// create a second collection, sharing the same configSet
String collectionNameWithSharedConfig = "SimpleCollectionCreateDeleteTest.collectionSharingAutocreatedConfigSet";
CollectionAdminRequest.Create createWithSharedConfig = CollectionAdminRequest.createCollection(collectionNameWithSharedConfig, configName, 1, 1);
NamedList<Object> requestWithSharedConfig = createWithSharedConfig.process(cloudClient).getResponse();
assertTrue("The collection with shared config set should have been created", requestWithSharedConfig.get("success") != null);
assertTrue("The new collection should exist after a successful creation", cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameWithSharedConfig, false));
String configNameOfSecondCollection = cloudClient.getZkStateReader().readConfigName(collectionNameWithSharedConfig);
assertEquals("Both collections should be using the same config", configName, configNameOfSecondCollection);
// delete the initial collection - the config set should stay, since it is shared with the other collection
CollectionAdminRequest.Delete deleteInitialCollection = CollectionAdminRequest.deleteCollection(collectionNameInitial);
cloudClient.request(deleteInitialCollection);
// initial collection has been deleted
assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameInitial, false));
// ... but not its autocreated config set, since it is shared with another collection
assertTrue("The auto-created config set should NOT have been deleted. Another collection is using it.", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
// delete the second collection - the config set should now be deleted, since it is no longer shared any other collection
CollectionAdminRequest.Delete deleteSecondCollection = CollectionAdminRequest.deleteCollection(collectionNameWithSharedConfig);
cloudClient.request(deleteSecondCollection);
// the collection has been deleted
assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameWithSharedConfig, false));
// ... and the config set is now also deleted - once it doesn't get referenced by any collection
assertFalse("The auto-created config set should have been deleted now. No collection is referencing it.", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
}
}
}