diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index de56036d713..83c949ca45e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -78,6 +78,12 @@ Upgrade Notes * SOLR-13541: Upgrade Jetty to 9.4.19.v20190610. (Erick Erickson, Cao Manh Dat) +* SOLR-13420: Routed Aliases now use collection properties rather than core properties to identify collections that + belong to the alias by default. This should be invisible and fully backwards compatible from within solr, and + existing routed alias collections with core based properties will continue to work, but new collections created will + not add a property to core.properties anymoore so any external code that inspected core.properties will not find the + 'routedAliasName' key in new cores belonging to routed aliases. + New Features ---------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java index 05cca40c470..752bc5b0ebe 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java @@ -25,6 +25,7 @@ import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.OverseerSolrResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.CollectionProperties; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams; @@ -64,7 +65,6 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd { "We require an explicit " + COLL_CONF); } createReqParams.set(NAME, createCollName); - createReqParams.set("property." + ROUTED_ALIAS_NAME_CORE_PROP, aliasName); // a CollectionOperation reads params and produces a message (Map) that is supposed to be sent to the Overseer. // Although we could create the Map without it, there are a fair amount of rules we don't want to reproduce. final Map createMsgMap = CollectionsHandler.CollectionOperation.CREATE_OP.execute( @@ -88,6 +88,11 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd { CollectionsHandler.waitForActiveCollection(createCollName, ocmh.overseer.getCoreContainer(), new OverseerSolrResponse(results)); + CollectionProperties collectionProperties = new CollectionProperties(ocmh.zkStateReader.getZkClient()); + collectionProperties.setCollectionProperty(createCollName,ROUTED_ALIAS_NAME_CORE_PROP,aliasName); + while (!ocmh.zkStateReader.getCollectionProperties(createCollName,1000).containsKey(ROUTED_ALIAS_NAME_CORE_PROP)) { + Thread.sleep(50); + } return results; } diff --git a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java index 13dd731b53f..384fcd81b71 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.api.collections.RoutedAlias; import org.apache.solr.common.SolrException; @@ -33,10 +35,12 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkCoreNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.UpdateParams; import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.SchemaField; @@ -67,6 +71,11 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor { private static final String ALIAS_DISTRIB_UPDATE_PARAM = "alias." + DISTRIB_UPDATE_PARAM; // param private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + // make sure we don't request collection properties any more frequently than once a minute during + // slow continuous indexing, and even less frequently during bulk indexing. (cache is updated by zk + // watch instead of re-requested until indexing has been stopped for the duration specified here) + public static final int CACHE_FOR_MILLIS = 60000; + // refs to std infrastructure private final SolrQueryRequest req; private final SolrCmdDistributor cmdDistrib; @@ -79,9 +88,24 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor { public static UpdateRequestProcessor wrap(SolrQueryRequest req, UpdateRequestProcessor next) { - //TODO get from "Collection property" - final String aliasName = req.getCore().getCoreDescriptor() - .getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null); + String aliasName = null; + // Demeter please don't arrest us... hide your eyes :( + // todo: a core should have a more direct way of finding a collection name, and the collection properties + SolrCore core = req.getCore(); + CoreDescriptor coreDescriptor = core.getCoreDescriptor(); + CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor(); + if (cloudDescriptor != null) { + String collectionName = cloudDescriptor.getCollectionName(); + CoreContainer coreContainer = core.getCoreContainer(); + ZkController zkController = coreContainer.getZkController(); + ZkStateReader zkStateReader = zkController.getZkStateReader(); + Map collectionProperties = zkStateReader.getCollectionProperties(collectionName, CACHE_FOR_MILLIS); + aliasName = collectionProperties.get(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP); + } + // fall back on core properties (legacy) + if (StringUtils.isBlank(aliasName)) { + aliasName = coreDescriptor.getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null); + } final DistribPhase shardDistribPhase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)); final DistribPhase aliasDistribPhase = diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java index c1d25a64589..6636bb6afdd 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -36,13 +37,13 @@ import org.apache.solr.client.solrj.request.ConfigSetAdminRequest; import org.apache.solr.client.solrj.response.FieldStatsInfo; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.api.collections.RoutedAlias; import org.apache.solr.cloud.api.collections.TimeRoutedAlias; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.update.UpdateCommand; @@ -54,6 +55,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.solr.cloud.api.collections.RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTIONS_ZKNODE; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROPS_ZKNODE; @LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13059") public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcessorTest { @@ -98,7 +102,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess final String col23rd = alias + "_2017-10-23"; CollectionAdminRequest.createCollection(col23rd, configName, 2, 2) .setMaxShardsPerNode(2) - .withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias) + .withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias) .process(solrClient); cluster.waitForActiveCollection(col23rd, 2, 4); @@ -132,7 +136,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess // destined for this collection, Solr will see it already exists and add it to the alias. final String col24th = alias + "_2017-10-24"; CollectionAdminRequest.createCollection(col24th, configName, 1, 1) // more shards and replicas now - .withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias) + .withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias) .process(solrClient); // index 3 documents in a random fashion @@ -175,6 +179,17 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess ); assertInvariants(alias + "_2017-10-26", alias + "_2017-10-25", col24th); + // verify that collection properties are set when the collections are created. Note: first 2 collections in + // this test have a core property instead, of a collection property but that MUST continue to work as well + // for back compatibility's reasons. + Thread.sleep(1000); + byte[] data = cluster.getZkClient() + .getData(COLLECTIONS_ZKNODE + "/" + alias + "_2017-10-26" + "/" + COLLECTION_PROPS_ZKNODE,null, null, true); + assertNotNull(data); + assertTrue(data.length > 0); + Map props = (Map) Utils.fromJSON(data); + assertTrue(props.containsKey(ROUTED_ALIAS_NAME_CORE_PROP)); + assertEquals(alias,props.get(ROUTED_ALIAS_NAME_CORE_PROP)); // update metadata to auto-delete oldest collections CollectionAdminRequest.setAliasProperty(alias)