Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-13420 Routed Aliases now use collection properties instead of core properties

This commit is contained in:
Gus Heck 2019-06-14 13:51:16 -04:00
parent 8d48f9252f
commit 5d550a34a9
4 changed files with 57 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
solr/CHANGES.txt
View File

@ -78,6 +78,12 @@ Upgrade Notes
* SOLR-13541: Upgrade Jetty to 9.4.19.v20190610. (Erick Erickson, Cao Manh Dat)
* SOLR-13420: Routed Aliases now use collection properties rather than core properties to identify collections that
belong to the alias by default. This should be invisible and fully backwards compatible from within solr, and
existing routed alias collections with core based properties will continue to work, but new collections created will
not add a property to core.properties anymoore so any external code that inspected core.properties will not find the
'routedAliasName' key in new cores belonging to routed aliases.
New Features
----------------------

7
solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
View File

@ -25,6 +25,7 @@ import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.OverseerSolrResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.CollectionProperties;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
@ -64,7 +65,6 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd {
"We require an explicit " + COLL_CONF);
}
createReqParams.set(NAME, createCollName);
createReqParams.set("property." + ROUTED_ALIAS_NAME_CORE_PROP, aliasName);
// a CollectionOperation reads params and produces a message (Map) that is supposed to be sent to the Overseer.
// Although we could create the Map without it, there are a fair amount of rules we don't want to reproduce.
final Map<String, Object> createMsgMap = CollectionsHandler.CollectionOperation.CREATE_OP.execute(
@ -88,6 +88,11 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd {
CollectionsHandler.waitForActiveCollection(createCollName, ocmh.overseer.getCoreContainer(),
new OverseerSolrResponse(results));
CollectionProperties collectionProperties = new CollectionProperties(ocmh.zkStateReader.getZkClient());
collectionProperties.setCollectionProperty(createCollName,ROUTED_ALIAS_NAME_CORE_PROP,aliasName);
while (!ocmh.zkStateReader.getCollectionProperties(createCollName,1000).containsKey(ROUTED_ALIAS_NAME_CORE_PROP)) {
Thread.sleep(50);
}
return results;
}

30
solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
View File

@ -24,6 +24,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.api.collections.RoutedAlias;
import org.apache.solr.common.SolrException;
@ -33,10 +35,12 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
@ -67,6 +71,11 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor {
private static final String ALIAS_DISTRIB_UPDATE_PARAM = "alias." + DISTRIB_UPDATE_PARAM; // param
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
// make sure we don't request collection properties any more frequently than once a minute during
// slow continuous indexing, and even less frequently during bulk indexing. (cache is updated by zk
// watch instead of re-requested until indexing has been stopped for the duration specified here)
public static final int CACHE_FOR_MILLIS = 60000;
// refs to std infrastructure
private final SolrQueryRequest req;
private final SolrCmdDistributor cmdDistrib;
@ -79,9 +88,24 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor {
public static UpdateRequestProcessor wrap(SolrQueryRequest req, UpdateRequestProcessor next) {
//TODO get from "Collection property"
final String aliasName = req.getCore().getCoreDescriptor()
.getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null);
String aliasName = null;
// Demeter please don't arrest us... hide your eyes :(
// todo: a core should have a more direct way of finding a collection name, and the collection properties
SolrCore core = req.getCore();
CoreDescriptor coreDescriptor = core.getCoreDescriptor();
CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
if (cloudDescriptor != null) {
String collectionName = cloudDescriptor.getCollectionName();
CoreContainer coreContainer = core.getCoreContainer();
ZkController zkController = coreContainer.getZkController();
ZkStateReader zkStateReader = zkController.getZkStateReader();
Map<String, String> collectionProperties = zkStateReader.getCollectionProperties(collectionName, CACHE_FOR_MILLIS);
aliasName = collectionProperties.get(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP);
}
// fall back on core properties (legacy)
if (StringUtils.isBlank(aliasName)) {
aliasName = coreDescriptor.getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null);
}
final DistribPhase shardDistribPhase =
DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
final DistribPhase aliasDistribPhase =

21
solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
View File

@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@ -36,13 +37,13 @@ import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.cloud.api.collections.RoutedAlias;
import org.apache.solr.cloud.api.collections.TimeRoutedAlias;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.update.UpdateCommand;
@ -54,6 +55,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.solr.cloud.api.collections.RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTIONS_ZKNODE;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROPS_ZKNODE;
@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13059")
public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcessorTest {
@ -98,7 +102,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
final String col23rd = alias + "_2017-10-23";
CollectionAdminRequest.createCollection(col23rd, configName, 2, 2)
.setMaxShardsPerNode(2)
.withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias)
.withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias)
.process(solrClient);
cluster.waitForActiveCollection(col23rd, 2, 4);
@ -132,7 +136,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
// destined for this collection, Solr will see it already exists and add it to the alias.
final String col24th = alias + "_2017-10-24";
CollectionAdminRequest.createCollection(col24th, configName, 1, 1) // more shards and replicas now
.withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias)
.withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias)
.process(solrClient);
// index 3 documents in a random fashion
@ -175,6 +179,17 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
);
assertInvariants(alias + "_2017-10-26", alias + "_2017-10-25", col24th);
// verify that collection properties are set when the collections are created. Note: first 2 collections in
// this test have a core property instead, of a collection property but that MUST continue to work as well
// for back compatibility's reasons.
Thread.sleep(1000);
byte[] data = cluster.getZkClient()
.getData(COLLECTIONS_ZKNODE + "/" + alias + "_2017-10-26" + "/" + COLLECTION_PROPS_ZKNODE,null, null, true);
assertNotNull(data);
assertTrue(data.length > 0);
Map<String,String> props = (Map<String, String>) Utils.fromJSON(data);
assertTrue(props.containsKey(ROUTED_ALIAS_NAME_CORE_PROP));
assertEquals(alias,props.get(ROUTED_ALIAS_NAME_CORE_PROP));
// update metadata to auto-delete oldest collections
CollectionAdminRequest.setAliasProperty(alias)