From 183835ed2485915006746e456d7124cb5d5d4abb Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Mon, 22 Jan 2018 19:48:41 +0530 Subject: [PATCH] SOLR-11624: Autocreated configsets will not use .AUTOCREATED suffix --- solr/CHANGES.txt | 10 +++++ .../api/collections/CreateCollectionCmd.java | 21 +++++----- .../handler/admin/ConfigSetsHandlerApi.java | 6 +++ .../TimeRoutedAliasUpdateProcessorTest.java | 40 ++++++++++++++----- solr/solr-ref-guide/src/collections-api.adoc | 2 +- 5 files changed, 55 insertions(+), 24 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9d3cbcbcd66..0291a5f3d6a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -67,6 +67,10 @@ Upgrade Notes Previously, the cool down period was a fixed period started after actions for a trigger event finish executing. During the cool down period, triggers wo +* SOLR-11624: Collections created without specifying a configset name use a copy of the _default configset since 7.0. + Before 7.3, the copied over configset was named the same as the collection name, but 7.3 onwards it will be named + with an additional ".AUTOCREATED" suffix. + New Features ---------------------- * SOLR-11285: Simulation framework for autoscaling. (ab) @@ -165,6 +169,12 @@ Other Changes * SOLR-11871: MoveReplicaSuggester should not suggest leader if other replicas are available (noble) +* SOLR-11624: Collections created from _default configset will now be associated with a configset with a suffix + .AUTOCREATED. For example, a new collection "mycollection", created without specifying a configset name, will + use the _default configset and the associated configset name will be "mycollection.AUTOCREATED". If this + collection is deleted and re-created, the autocreated configset will be left behind and will be re-used for + the re-created collection (Ishan Chattopadhyaya, Abhishek Kumar Singh) + ================== 7.2.1 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index 4d9c9712261..d5ceb6a400e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -350,10 +350,13 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd try { configNames = ocmh.zkStateReader.getZkClient().getChildren(ZkConfigManager.CONFIGS_ZKNODE, null, true); if (configNames.contains(ConfigSetsHandlerApi.DEFAULT_CONFIGSET_NAME)) { - if (!CollectionAdminParams.SYSTEM_COLL.equals(coll)) { - copyDefaultConfigSetTo(configNames, coll); + if (CollectionAdminParams.SYSTEM_COLL.equals(coll)) { + return coll; + } else { + String intendedConfigSetName = ConfigSetsHandlerApi.getSuffixedNameForAutoGeneratedConfigSet(coll); + copyDefaultConfigSetTo(configNames, intendedConfigSetName); + return intendedConfigSetName; } - return coll; } else if (configNames != null && configNames.size() == 1) { configName = configNames.get(0); // no config set named, but there is only 1 - use it @@ -372,17 +375,11 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd private void copyDefaultConfigSetTo(List configNames, String targetConfig) { ZkConfigManager configManager = new ZkConfigManager(ocmh.zkStateReader.getZkClient()); - // if a configset named coll exists, delete the configset so that _default can be copied over + // if a configset named collection exists, re-use it if (configNames.contains(targetConfig)) { log.info("There exists a configset by the same name as the collection we're trying to create: " + targetConfig + - ", deleting it so that we can copy the _default configs over and create the collection."); - try { - configManager.deleteConfigDir(targetConfig); - } catch (Exception e) { - throw new SolrException(ErrorCode.INVALID_STATE, "Error while deleting configset: " + targetConfig, e); - } - } else { - log.info("Only _default config set found, using it."); + ", re-using it."); + return; } // Copy _default into targetConfig try { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java index 2028f677c41..1a5f6f33621 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java @@ -32,10 +32,16 @@ import org.apache.solr.response.SolrQueryResponse; public class ConfigSetsHandlerApi extends BaseHandlerApiSupport { final public static String DEFAULT_CONFIGSET_NAME = "_default"; + final public static String AUTOCREATED_CONFIGSET_SUFFIX = ".AUTOCREATED"; + final ConfigSetsHandler configSetHandler; static Collection apiCommands = createMapping(); + public static String getSuffixedNameForAutoGeneratedConfigSet(String configName) { + return configName + AUTOCREATED_CONFIGSET_SUFFIX; + } + private static Collection createMapping() { Map result = new EnumMap<>(ConfigSetMeta.class); diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java index 102377d7687..f524fdae3fb 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java @@ -36,6 +36,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.ConfigSetAdminRequest; import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.response.ConfigSetAdminResponse; import org.apache.solr.client.solrj.response.FieldStatsInfo; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; @@ -78,14 +79,21 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase { @Test public void test() throws Exception { - // First create a config using REST API. To do this, we create a collection with the name of the eventual config. - // We configure it, and ultimately delete it the collection, leaving a config with the same name behind. - // Then when we create the "real" collections referencing this config. - CollectionAdminRequest.createCollection(configName, 1, 1).process(solrClient); + + // First create a configSet + // Then we create a collection with the name of the eventual config. + // We configure it, and ultimately delete the collection, leaving a modified config-set behind. + // Then when we create the "real" collections referencing this modified config-set. + final ConfigSetAdminRequest.Create adminRequest = new ConfigSetAdminRequest.Create(); + adminRequest.setConfigSetName(configName); + adminRequest.setBaseConfigSetName("_default"); + ConfigSetAdminResponse adminResponse = adminRequest.process(solrClient); + assertEquals(adminResponse.getStatus(), 0); + + CollectionAdminRequest.createCollection(configName, configName,1, 1).process(solrClient); // manipulate the config... - checkNoError(solrClient.request(new V2Request.Builder("/collections/" + configName + "/config") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{" + + + String conf = "{" + " 'set-user-property' : {'timePartitionAliasName':'" + alias + "'}," + // no data driven " 'set-user-property' : {'update.autoCreateFields':false}," + // no data driven " 'add-updateprocessor' : {" + @@ -95,8 +103,10 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase { " 'name':'inc', 'class':'" + IncrementURPFactory.class.getName() + "'," + " 'fieldName':'" + intField + "'" + " }," + - "}").build())); - // only sometimes test with "tolerant" URP + "}"; + checkNoError(solrClient.request(new V2Request.Builder("/collections/" + configName + "/config") + .withMethod(SolrRequest.METHOD.POST) + .withPayload(conf).build())); // only sometimes test with "tolerant" URP final String urpNames = "inc" + (random().nextBoolean() ? ",tolerant" : ""); checkNoError(solrClient.request(new V2Request.Builder("/collections/" + configName + "/config/params") .withMethod(SolrRequest.METHOD.POST) @@ -107,6 +117,11 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase { "}").build())); CollectionAdminRequest.deleteCollection(configName).process(solrClient); + assertTrue( + new ConfigSetAdminRequest.List().process(solrClient).getConfigSets() + .contains(configName) + ); + // start with one collection and an alias for it final String col23rd = alias + "_2017-10-23"; CollectionAdminRequest.createCollection(col23rd, configName, 2, 2) @@ -114,8 +129,11 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase { .withProperty(TimeRoutedAliasUpdateProcessor.TIME_PARTITION_ALIAS_NAME_CORE_PROP, alias) .process(solrClient); - assertEquals("We only expect 2 configSets", - Arrays.asList("_default", configName), new ConfigSetAdminRequest.List().process(solrClient).getConfigSets()); + List retrievedConfigSetNames = new ConfigSetAdminRequest.List().process(solrClient).getConfigSets(); + List expectedConfigSetNames = Arrays.asList("_default", configName); + assertTrue("We only expect 2 configSets", + expectedConfigSetNames.size() == retrievedConfigSetNames.size()); + assertTrue("ConfigNames should be :" + expectedConfigSetNames, expectedConfigSetNames.containsAll(retrievedConfigSetNames) && retrievedConfigSetNames.containsAll(expectedConfigSetNames)); CollectionAdminRequest.createAlias(alias, col23rd).process(solrClient); //TODO use SOLR-11617 client API to set alias metadata diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc index a0c8038951e..8a48ba06ecc 100644 --- a/solr/solr-ref-guide/src/collections-api.adoc +++ b/solr/solr-ref-guide/src/collections-api.adoc @@ -85,7 +85,7 @@ A `false` value makes the results of a collection creation predictable and gives This parameter is ignored if `createNodeSet` is not also specified. `collection.configName`:: -Defines the name of the configurations (which *must already be stored in ZooKeeper*) to use for this collection. If not provided, Solr will default to the collection name as the configuration name. +Defines the name of the configuration (which *must already be stored in ZooKeeper*) to use for this collection. If not provided, Solr will use the configuration of `_default` configSet to create a new (and mutable) configSet named `.AUTOCREATED` and will use it for the new collection. When such a collection (that uses a copy of the _default configset) is deleted, the autocreated configset is not deleted by default. `router.field`:: If this parameter is specified, the router will look at the value of the field in an input document to compute the hash and identify a shard instead of looking at the `uniqueKey` field. If the field specified is null in the document, the document will be rejected.