From ed137dbe281cfb314af340673a7b646922a2e7d1 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Thu, 8 Aug 2019 15:14:55 +0530 Subject: [PATCH] SOLR-13674: NodeAddedTrigger does not support configuration of replica type hint. A new replicaType property has been added to NodeAddTrigger so that new replicas of the given type are added when the preferredOp is addreplica. The default value of replicaType is `NRT`. This closes #821. --- .gitignore | 1 + solr/CHANGES.txt | 4 ++ .../cloud/autoscaling/ComputePlanAction.java | 3 ++ .../cloud/autoscaling/NodeAddedTrigger.java | 17 ++++-- .../autoscaling/ComputePlanActionTest.java | 53 ++++++++++++++++--- .../src/solrcloud-autoscaling-triggers.adoc | 7 ++- .../solr/common/params/AutoScalingParams.java | 1 + 7 files changed, 75 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 1a3e1dbf4e9..d0f0adedeb3 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ pom.xml .pydevproject __pycache__ /dev-tools/scripts/scripts.iml +.DS_Store diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 33eacb6493d..f0018a0a57b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -136,6 +136,10 @@ Bug Fixes * SOLR-13672: Cloud -> Zk Status page now parses response from Zookeeper 3.5.5 correctly (Jörn Franke, janhoy, Shawn Heisey) +* SOLR-13674: NodeAddedTrigger does not support configuration of replica type hint. A new replicaType property + has been added to NodeAddTrigger so that new replicas of the given type are added. The default value is `NRT`. + (Irena Shaigorodsky via shalin) + Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java index 7103bf56944..cac993dbd11 100644 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java +++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java @@ -39,6 +39,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.UnsupportedSuggester; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.params.AutoScalingParams; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CoreAdminParams; @@ -269,6 +270,7 @@ public class ComputePlanAction extends TriggerActionBase { private Suggester getNodeAddedSuggester(SolrCloudManager cloudManager, Policy.Session session, TriggerEvent event) throws IOException { String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); + Replica.Type replicaType = (Replica.Type) event.getProperty(AutoScalingParams.REPLICA_TYPE, Replica.Type.NRT); CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp); Suggester suggester = session.getSuggester(action) @@ -289,6 +291,7 @@ public class ComputePlanAction extends TriggerActionBase { } }); suggester.hint(Suggester.Hint.COLL_SHARD, collShards); + suggester.hint(Suggester.Hint.REPLICATYPE, replicaType); break; case MOVEREPLICA: case NONE: diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java index b05e8fd9408..6b87fc32382 100644 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java +++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.cloud.SolrCloudManager; import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.core.SolrResourceLoader; @@ -40,6 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.solr.common.params.AutoScalingParams.PREFERRED_OP; +import static org.apache.solr.common.params.AutoScalingParams.REPLICA_TYPE; /** * Trigger for the {@link TriggerEventType#NODEADDED} event @@ -52,10 +54,11 @@ public class NodeAddedTrigger extends TriggerBase { private Map nodeNameVsTimeAdded = new HashMap<>(); private String preferredOp; + private Replica.Type replicaType = Replica.Type.NRT; public NodeAddedTrigger(String name) { super(TriggerEventType.NODEADDED, name); - TriggerUtils.validProperties(validProperties, PREFERRED_OP); + TriggerUtils.validProperties(validProperties, PREFERRED_OP, REPLICA_TYPE); } @Override @@ -87,7 +90,14 @@ public class NodeAddedTrigger extends TriggerBase { super.configure(loader, cloudManager, properties); preferredOp = (String) properties.getOrDefault(PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); preferredOp = preferredOp.toLowerCase(Locale.ROOT); + String replicaTypeStr = (String) properties.getOrDefault(REPLICA_TYPE, Replica.Type.NRT.name()); // verify + try { + replicaType = Replica.Type.valueOf(replicaTypeStr); + } catch (IllegalArgumentException | NullPointerException e) { + throw new TriggerValidationException("Unsupported replicaType=" + replicaTypeStr + " specified for node added trigger"); + } + CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp); switch (action) { case ADDREPLICA: @@ -183,7 +193,7 @@ public class NodeAddedTrigger extends TriggerBase { if (processor != null) { log.debug("NodeAddedTrigger {} firing registered processor for nodes: {} added at times {}, now={}", name, nodeNames, times, cloudManager.getTimeSource().getTimeNs()); - if (processor.process(new NodeAddedEvent(getEventType(), getName(), times, nodeNames, preferredOp))) { + if (processor.process(new NodeAddedEvent(getEventType(), getName(), times, nodeNames, preferredOp, replicaType))) { // remove from tracking set only if the fire was accepted nodeNames.forEach(n -> { log.debug("Removing new node from tracking: {}", n); @@ -206,12 +216,13 @@ public class NodeAddedTrigger extends TriggerBase { public static class NodeAddedEvent extends TriggerEvent { - public NodeAddedEvent(TriggerEventType eventType, String source, List times, List nodeNames, String preferredOp) { + public NodeAddedEvent(TriggerEventType eventType, String source, List times, List nodeNames, String preferredOp, Replica.Type replicaType) { // use the oldest time as the time of the event super(eventType, source, times.get(0), null); properties.put(NODE_NAMES, nodeNames); properties.put(EVENT_TIMES, times); properties.put(PREFERRED_OP, preferredOp); + properties.put(REPLICA_TYPE, replicaType); } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java index 47be3dd4096..9b579ac0ce6 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java @@ -524,6 +524,15 @@ public class ComputePlanActionTest extends SolrCloudTestCase { nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections); } + @Test + public void testNodeAddedTriggerWithAddReplicaPreferredOpReplicaType_1Shard() throws Exception { + String collectionNamePrefix = "testNodeAddedTriggerWithAddReplicaPreferredOpReplicaType_1Shard"; + int numShards = 1; + int numCollections = 5; + + nodeAddedTriggerWithAddReplicaPreferredOpReplicaType(collectionNamePrefix, numShards, numCollections); + } + @Test // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 public void testNodeAddedTriggerWithAddReplicaPreferredOp_2Shard() throws Exception { @@ -533,9 +542,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase { nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections); } - private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections) throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); String setTriggerCommand = "{" + "'set-trigger' : {" + "'name' : 'node_added_trigger'," + @@ -546,9 +553,6 @@ public class ComputePlanActionTest extends SolrCloudTestCase { "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" + "}}"; - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); String setClusterPolicyCommand = "{" + " 'set-cluster-policy': [" + @@ -557,13 +561,50 @@ public class ComputePlanActionTest extends SolrCloudTestCase { " {'nodeRole':'overseer', 'replica':0}" + " ]" + "}"; + + nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand); + } + + private void nodeAddedTriggerWithAddReplicaPreferredOpReplicaType(String collectionNamePrefix, int numShards, int numCollections) throws Exception { + String setTriggerCommand = "{" + + "'set-trigger' : {" + + "'name' : 'node_added_trigger'," + + "'event' : 'nodeAdded'," + + "'waitFor' : '1s'," + + "'enabled' : true," + + "'" + AutoScalingParams.PREFERRED_OP + "':'addreplica'," + + "'" + AutoScalingParams.REPLICA_TYPE + "':'" + Replica.Type.PULL + "'," + + "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + + "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" + + "}}"; + + String setClusterPolicyCommand = "{" + + " 'set-cluster-policy': [" + + " {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," + + " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + + " {'nodeRole':'overseer', 'replica':0}" + + " ]" + + "}"; + + nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand, 0, 1, 0); + } + + private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections, String setTriggerCommand, String setClusterPolicyCommand) throws Exception { + nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand, 1, null, null); + } + private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections, String setTriggerCommand, String setClusterPolicyCommand, Integer nNrtReplicas, Integer nTlogReplicas, Integer nPullReplicas) throws Exception { + CloudSolrClient solrClient = cluster.getSolrClient(); + SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); + NamedList response = solrClient.request(req); + assertEquals(response.get("result").toString(), "success"); + req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); response = solrClient.request(req); assertEquals(response.get("result").toString(), "success"); CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_0", - "conf", numShards, 1).setMaxShardsPerNode(2); + "conf", numShards, nNrtReplicas, nTlogReplicas, nPullReplicas).setMaxShardsPerNode(2); create.process(solrClient); waitForState("Timed out waiting for replicas of new collection to be active", diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc index 85f273e9742..190381d7e37 100644 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc +++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc @@ -133,6 +133,8 @@ In addition to the parameters described at <>, this trigg `preferredOperation`:: (string, optional, defaults to `movereplica`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from other nodes to the added node. The only other supported value is `addreplica` which adds more replicas of the existing collections on the new node. +`replicaType`:: (string, optional, defaults to `NRT`) The replica type that will be used to add replicas, in response to an event generated by this trigger, when "preferredOperation" is "ADDREPLICA". By default, the replica(s) will be of type `NRT`. The only other supported values are `PULL` and `TLOG`, which will add more replicas of the specified type to the existing collections on the new node. + .Example: Node Added Trigger to move replicas to new node [source,json] ---- @@ -145,7 +147,7 @@ In addition to the parameters described at <>, this trigg } ---- -.Example: Node Added Trigger to add replicas on new node +.Example: Node Added Trigger to add replicas on new node with replica type PULL [source,json] ---- { @@ -153,7 +155,8 @@ In addition to the parameters described at <>, this trigg "name": "node_added_trigger", "event": "nodeAdded", "waitFor": "5s", - "preferredOperation": "ADDREPLICA" + "preferredOperation": "ADDREPLICA", + "replicaType": "PULL" } } ---- diff --git a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java b/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java index 0d4cc89f333..e626fef1827 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java @@ -50,6 +50,7 @@ public interface AutoScalingParams { String ABOVE = "above"; String BELOW = "below"; String PREFERRED_OP = "preferredOperation"; + String REPLICA_TYPE = "replicaType"; // commands String CMD_SET_TRIGGER = "set-trigger";